radeon.h 87 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812
  1. /*
  2. * Copyright 2008 Advanced Micro Devices, Inc.
  3. * Copyright 2008 Red Hat Inc.
  4. * Copyright 2009 Jerome Glisse.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the "Software"),
  8. * to deal in the Software without restriction, including without limitation
  9. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10. * and/or sell copies of the Software, and to permit persons to whom the
  11. * Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors: Dave Airlie
  25. * Alex Deucher
  26. * Jerome Glisse
  27. */
  28. #ifndef __RADEON_H__
  29. #define __RADEON_H__
  30. /* TODO: Here are things that needs to be done :
  31. * - surface allocator & initializer : (bit like scratch reg) should
  32. * initialize HDP_ stuff on RS600, R600, R700 hw, well anythings
  33. * related to surface
  34. * - WB : write back stuff (do it bit like scratch reg things)
  35. * - Vblank : look at Jesse's rework and what we should do
  36. * - r600/r700: gart & cp
  37. * - cs : clean cs ioctl use bitmap & things like that.
  38. * - power management stuff
  39. * - Barrier in gart code
  40. * - Unmappabled vram ?
  41. * - TESTING, TESTING, TESTING
  42. */
  43. /* Initialization path:
  44. * We expect that acceleration initialization might fail for various
  45. * reasons even thought we work hard to make it works on most
  46. * configurations. In order to still have a working userspace in such
  47. * situation the init path must succeed up to the memory controller
  48. * initialization point. Failure before this point are considered as
  49. * fatal error. Here is the init callchain :
  50. * radeon_device_init perform common structure, mutex initialization
  51. * asic_init setup the GPU memory layout and perform all
  52. * one time initialization (failure in this
  53. * function are considered fatal)
  54. * asic_startup setup the GPU acceleration, in order to
  55. * follow guideline the first thing this
  56. * function should do is setting the GPU
  57. * memory controller (only MC setup failure
  58. * are considered as fatal)
  59. */
  60. #include <linux/atomic.h>
  61. #include <linux/wait.h>
  62. #include <linux/list.h>
  63. #include <linux/kref.h>
  64. #include <ttm/ttm_bo_api.h>
  65. #include <ttm/ttm_bo_driver.h>
  66. #include <ttm/ttm_placement.h>
  67. #include <ttm/ttm_module.h>
  68. #include <ttm/ttm_execbuf_util.h>
  69. #include "radeon_family.h"
  70. #include "radeon_mode.h"
  71. #include "radeon_reg.h"
  72. /*
  73. * Modules parameters.
  74. */
  75. extern int radeon_no_wb;
  76. extern int radeon_modeset;
  77. extern int radeon_dynclks;
  78. extern int radeon_r4xx_atom;
  79. extern int radeon_agpmode;
  80. extern int radeon_vram_limit;
  81. extern int radeon_gart_size;
  82. extern int radeon_benchmarking;
  83. extern int radeon_testing;
  84. extern int radeon_connector_table;
  85. extern int radeon_tv;
  86. extern int radeon_audio;
  87. extern int radeon_disp_priority;
  88. extern int radeon_hw_i2c;
  89. extern int radeon_pcie_gen2;
  90. extern int radeon_msi;
  91. extern int radeon_lockup_timeout;
  92. extern int radeon_fastfb;
  93. extern int radeon_dpm;
  94. extern int radeon_aspm;
  95. extern int radeon_runtime_pm;
  96. /*
  97. * Copy from radeon_drv.h so we don't have to include both and have conflicting
  98. * symbol;
  99. */
  100. #define RADEON_MAX_USEC_TIMEOUT 100000 /* 100 ms */
  101. #define RADEON_FENCE_JIFFIES_TIMEOUT (HZ / 2)
  102. /* RADEON_IB_POOL_SIZE must be a power of 2 */
  103. #define RADEON_IB_POOL_SIZE 16
  104. #define RADEON_DEBUGFS_MAX_COMPONENTS 32
  105. #define RADEONFB_CONN_LIMIT 4
  106. #define RADEON_BIOS_NUM_SCRATCH 8
  107. /* max number of rings */
  108. #define RADEON_NUM_RINGS 6
  109. /* fence seq are set to this number when signaled */
  110. #define RADEON_FENCE_SIGNALED_SEQ 0LL
  111. /* internal ring indices */
  112. /* r1xx+ has gfx CP ring */
  113. #define RADEON_RING_TYPE_GFX_INDEX 0
  114. /* cayman has 2 compute CP rings */
  115. #define CAYMAN_RING_TYPE_CP1_INDEX 1
  116. #define CAYMAN_RING_TYPE_CP2_INDEX 2
  117. /* R600+ has an async dma ring */
  118. #define R600_RING_TYPE_DMA_INDEX 3
  119. /* cayman add a second async dma ring */
  120. #define CAYMAN_RING_TYPE_DMA1_INDEX 4
  121. /* R600+ */
  122. #define R600_RING_TYPE_UVD_INDEX 5
  123. /* hardcode those limit for now */
  124. #define RADEON_VA_IB_OFFSET (1 << 20)
  125. #define RADEON_VA_RESERVED_SIZE (8 << 20)
  126. #define RADEON_IB_VM_MAX_SIZE (64 << 10)
  127. /* reset flags */
  128. #define RADEON_RESET_GFX (1 << 0)
  129. #define RADEON_RESET_COMPUTE (1 << 1)
  130. #define RADEON_RESET_DMA (1 << 2)
  131. #define RADEON_RESET_CP (1 << 3)
  132. #define RADEON_RESET_GRBM (1 << 4)
  133. #define RADEON_RESET_DMA1 (1 << 5)
  134. #define RADEON_RESET_RLC (1 << 6)
  135. #define RADEON_RESET_SEM (1 << 7)
  136. #define RADEON_RESET_IH (1 << 8)
  137. #define RADEON_RESET_VMC (1 << 9)
  138. #define RADEON_RESET_MC (1 << 10)
  139. #define RADEON_RESET_DISPLAY (1 << 11)
  140. /* CG block flags */
  141. #define RADEON_CG_BLOCK_GFX (1 << 0)
  142. #define RADEON_CG_BLOCK_MC (1 << 1)
  143. #define RADEON_CG_BLOCK_SDMA (1 << 2)
  144. #define RADEON_CG_BLOCK_UVD (1 << 3)
  145. #define RADEON_CG_BLOCK_VCE (1 << 4)
  146. #define RADEON_CG_BLOCK_HDP (1 << 5)
  147. #define RADEON_CG_BLOCK_BIF (1 << 6)
  148. /* CG flags */
  149. #define RADEON_CG_SUPPORT_GFX_MGCG (1 << 0)
  150. #define RADEON_CG_SUPPORT_GFX_MGLS (1 << 1)
  151. #define RADEON_CG_SUPPORT_GFX_CGCG (1 << 2)
  152. #define RADEON_CG_SUPPORT_GFX_CGLS (1 << 3)
  153. #define RADEON_CG_SUPPORT_GFX_CGTS (1 << 4)
  154. #define RADEON_CG_SUPPORT_GFX_CGTS_LS (1 << 5)
  155. #define RADEON_CG_SUPPORT_GFX_CP_LS (1 << 6)
  156. #define RADEON_CG_SUPPORT_GFX_RLC_LS (1 << 7)
  157. #define RADEON_CG_SUPPORT_MC_LS (1 << 8)
  158. #define RADEON_CG_SUPPORT_MC_MGCG (1 << 9)
  159. #define RADEON_CG_SUPPORT_SDMA_LS (1 << 10)
  160. #define RADEON_CG_SUPPORT_SDMA_MGCG (1 << 11)
  161. #define RADEON_CG_SUPPORT_BIF_LS (1 << 12)
  162. #define RADEON_CG_SUPPORT_UVD_MGCG (1 << 13)
  163. #define RADEON_CG_SUPPORT_VCE_MGCG (1 << 14)
  164. #define RADEON_CG_SUPPORT_HDP_LS (1 << 15)
  165. #define RADEON_CG_SUPPORT_HDP_MGCG (1 << 16)
  166. /* PG flags */
  167. #define RADEON_PG_SUPPORT_GFX_PG (1 << 0)
  168. #define RADEON_PG_SUPPORT_GFX_SMG (1 << 1)
  169. #define RADEON_PG_SUPPORT_GFX_DMG (1 << 2)
  170. #define RADEON_PG_SUPPORT_UVD (1 << 3)
  171. #define RADEON_PG_SUPPORT_VCE (1 << 4)
  172. #define RADEON_PG_SUPPORT_CP (1 << 5)
  173. #define RADEON_PG_SUPPORT_GDS (1 << 6)
  174. #define RADEON_PG_SUPPORT_RLC_SMU_HS (1 << 7)
  175. #define RADEON_PG_SUPPORT_SDMA (1 << 8)
  176. #define RADEON_PG_SUPPORT_ACP (1 << 9)
  177. #define RADEON_PG_SUPPORT_SAMU (1 << 10)
  178. /* max cursor sizes (in pixels) */
  179. #define CURSOR_WIDTH 64
  180. #define CURSOR_HEIGHT 64
  181. #define CIK_CURSOR_WIDTH 128
  182. #define CIK_CURSOR_HEIGHT 128
  183. /*
  184. * Errata workarounds.
  185. */
  186. enum radeon_pll_errata {
  187. CHIP_ERRATA_R300_CG = 0x00000001,
  188. CHIP_ERRATA_PLL_DUMMYREADS = 0x00000002,
  189. CHIP_ERRATA_PLL_DELAY = 0x00000004
  190. };
  191. struct radeon_device;
  192. /*
  193. * BIOS.
  194. */
  195. bool radeon_get_bios(struct radeon_device *rdev);
  196. /*
  197. * Dummy page
  198. */
  199. struct radeon_dummy_page {
  200. struct page *page;
  201. dma_addr_t addr;
  202. };
  203. int radeon_dummy_page_init(struct radeon_device *rdev);
  204. void radeon_dummy_page_fini(struct radeon_device *rdev);
  205. /*
  206. * Clocks
  207. */
  208. struct radeon_clock {
  209. struct radeon_pll p1pll;
  210. struct radeon_pll p2pll;
  211. struct radeon_pll dcpll;
  212. struct radeon_pll spll;
  213. struct radeon_pll mpll;
  214. /* 10 Khz units */
  215. uint32_t default_mclk;
  216. uint32_t default_sclk;
  217. uint32_t default_dispclk;
  218. uint32_t current_dispclk;
  219. uint32_t dp_extclk;
  220. uint32_t max_pixel_clock;
  221. };
  222. /*
  223. * Power management
  224. */
  225. int radeon_pm_init(struct radeon_device *rdev);
  226. void radeon_pm_fini(struct radeon_device *rdev);
  227. void radeon_pm_compute_clocks(struct radeon_device *rdev);
  228. void radeon_pm_suspend(struct radeon_device *rdev);
  229. void radeon_pm_resume(struct radeon_device *rdev);
  230. void radeon_combios_get_power_modes(struct radeon_device *rdev);
  231. void radeon_atombios_get_power_modes(struct radeon_device *rdev);
  232. int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
  233. u8 clock_type,
  234. u32 clock,
  235. bool strobe_mode,
  236. struct atom_clock_dividers *dividers);
  237. int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev,
  238. u32 clock,
  239. bool strobe_mode,
  240. struct atom_mpll_param *mpll_param);
  241. void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type);
  242. int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev,
  243. u16 voltage_level, u8 voltage_type,
  244. u32 *gpio_value, u32 *gpio_mask);
  245. void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev,
  246. u32 eng_clock, u32 mem_clock);
  247. int radeon_atom_get_voltage_step(struct radeon_device *rdev,
  248. u8 voltage_type, u16 *voltage_step);
  249. int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
  250. u16 voltage_id, u16 *voltage);
  251. int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev,
  252. u16 *voltage,
  253. u16 leakage_idx);
  254. int radeon_atom_get_leakage_id_from_vbios(struct radeon_device *rdev,
  255. u16 *leakage_id);
  256. int radeon_atom_get_leakage_vddc_based_on_leakage_params(struct radeon_device *rdev,
  257. u16 *vddc, u16 *vddci,
  258. u16 virtual_voltage_id,
  259. u16 vbios_voltage_id);
  260. int radeon_atom_round_to_true_voltage(struct radeon_device *rdev,
  261. u8 voltage_type,
  262. u16 nominal_voltage,
  263. u16 *true_voltage);
  264. int radeon_atom_get_min_voltage(struct radeon_device *rdev,
  265. u8 voltage_type, u16 *min_voltage);
  266. int radeon_atom_get_max_voltage(struct radeon_device *rdev,
  267. u8 voltage_type, u16 *max_voltage);
  268. int radeon_atom_get_voltage_table(struct radeon_device *rdev,
  269. u8 voltage_type, u8 voltage_mode,
  270. struct atom_voltage_table *voltage_table);
  271. bool radeon_atom_is_voltage_gpio(struct radeon_device *rdev,
  272. u8 voltage_type, u8 voltage_mode);
  273. void radeon_atom_update_memory_dll(struct radeon_device *rdev,
  274. u32 mem_clock);
  275. void radeon_atom_set_ac_timing(struct radeon_device *rdev,
  276. u32 mem_clock);
  277. int radeon_atom_init_mc_reg_table(struct radeon_device *rdev,
  278. u8 module_index,
  279. struct atom_mc_reg_table *reg_table);
  280. int radeon_atom_get_memory_info(struct radeon_device *rdev,
  281. u8 module_index, struct atom_memory_info *mem_info);
  282. int radeon_atom_get_mclk_range_table(struct radeon_device *rdev,
  283. bool gddr5, u8 module_index,
  284. struct atom_memory_clock_range_table *mclk_range_table);
  285. int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
  286. u16 voltage_id, u16 *voltage);
  287. void rs690_pm_info(struct radeon_device *rdev);
  288. extern void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
  289. unsigned *bankh, unsigned *mtaspect,
  290. unsigned *tile_split);
  291. /*
  292. * Fences.
  293. */
  294. struct radeon_fence_driver {
  295. uint32_t scratch_reg;
  296. uint64_t gpu_addr;
  297. volatile uint32_t *cpu_addr;
  298. /* sync_seq is protected by ring emission lock */
  299. uint64_t sync_seq[RADEON_NUM_RINGS];
  300. atomic64_t last_seq;
  301. bool initialized;
  302. };
  303. struct radeon_fence {
  304. struct radeon_device *rdev;
  305. struct kref kref;
  306. /* protected by radeon_fence.lock */
  307. uint64_t seq;
  308. /* RB, DMA, etc. */
  309. unsigned ring;
  310. };
  311. int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring);
  312. int radeon_fence_driver_init(struct radeon_device *rdev);
  313. void radeon_fence_driver_fini(struct radeon_device *rdev);
  314. void radeon_fence_driver_force_completion(struct radeon_device *rdev);
  315. int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring);
  316. void radeon_fence_process(struct radeon_device *rdev, int ring);
  317. bool radeon_fence_signaled(struct radeon_fence *fence);
  318. int radeon_fence_wait(struct radeon_fence *fence, bool interruptible);
  319. int radeon_fence_wait_locked(struct radeon_fence *fence);
  320. int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring);
  321. int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring);
  322. int radeon_fence_wait_any(struct radeon_device *rdev,
  323. struct radeon_fence **fences,
  324. bool intr);
  325. struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence);
  326. void radeon_fence_unref(struct radeon_fence **fence);
  327. unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring);
  328. bool radeon_fence_need_sync(struct radeon_fence *fence, int ring);
  329. void radeon_fence_note_sync(struct radeon_fence *fence, int ring);
  330. static inline struct radeon_fence *radeon_fence_later(struct radeon_fence *a,
  331. struct radeon_fence *b)
  332. {
  333. if (!a) {
  334. return b;
  335. }
  336. if (!b) {
  337. return a;
  338. }
  339. BUG_ON(a->ring != b->ring);
  340. if (a->seq > b->seq) {
  341. return a;
  342. } else {
  343. return b;
  344. }
  345. }
  346. static inline bool radeon_fence_is_earlier(struct radeon_fence *a,
  347. struct radeon_fence *b)
  348. {
  349. if (!a) {
  350. return false;
  351. }
  352. if (!b) {
  353. return true;
  354. }
  355. BUG_ON(a->ring != b->ring);
  356. return a->seq < b->seq;
  357. }
  358. /*
  359. * Tiling registers
  360. */
  361. struct radeon_surface_reg {
  362. struct radeon_bo *bo;
  363. };
  364. #define RADEON_GEM_MAX_SURFACES 8
  365. /*
  366. * TTM.
  367. */
  368. struct radeon_mman {
  369. struct ttm_bo_global_ref bo_global_ref;
  370. struct drm_global_reference mem_global_ref;
  371. struct ttm_bo_device bdev;
  372. bool mem_global_referenced;
  373. bool initialized;
  374. };
  375. /* bo virtual address in a specific vm */
  376. struct radeon_bo_va {
  377. /* protected by bo being reserved */
  378. struct list_head bo_list;
  379. uint64_t soffset;
  380. uint64_t eoffset;
  381. uint32_t flags;
  382. bool valid;
  383. unsigned ref_count;
  384. /* protected by vm mutex */
  385. struct list_head vm_list;
  386. /* constant after initialization */
  387. struct radeon_vm *vm;
  388. struct radeon_bo *bo;
  389. };
  390. struct radeon_bo {
  391. /* Protected by gem.mutex */
  392. struct list_head list;
  393. /* Protected by tbo.reserved */
  394. u32 placements[3];
  395. struct ttm_placement placement;
  396. struct ttm_buffer_object tbo;
  397. struct ttm_bo_kmap_obj kmap;
  398. unsigned pin_count;
  399. void *kptr;
  400. u32 tiling_flags;
  401. u32 pitch;
  402. int surface_reg;
  403. /* list of all virtual address to which this bo
  404. * is associated to
  405. */
  406. struct list_head va;
  407. /* Constant after initialization */
  408. struct radeon_device *rdev;
  409. struct drm_gem_object gem_base;
  410. struct ttm_bo_kmap_obj dma_buf_vmap;
  411. pid_t pid;
  412. };
  413. #define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base)
  414. struct radeon_bo_list {
  415. struct ttm_validate_buffer tv;
  416. struct radeon_bo *bo;
  417. uint64_t gpu_offset;
  418. bool written;
  419. unsigned domain;
  420. unsigned alt_domain;
  421. u32 tiling_flags;
  422. };
  423. int radeon_gem_debugfs_init(struct radeon_device *rdev);
  424. /* sub-allocation manager, it has to be protected by another lock.
  425. * By conception this is an helper for other part of the driver
  426. * like the indirect buffer or semaphore, which both have their
  427. * locking.
  428. *
  429. * Principe is simple, we keep a list of sub allocation in offset
  430. * order (first entry has offset == 0, last entry has the highest
  431. * offset).
  432. *
  433. * When allocating new object we first check if there is room at
  434. * the end total_size - (last_object_offset + last_object_size) >=
  435. * alloc_size. If so we allocate new object there.
  436. *
  437. * When there is not enough room at the end, we start waiting for
  438. * each sub object until we reach object_offset+object_size >=
  439. * alloc_size, this object then become the sub object we return.
  440. *
  441. * Alignment can't be bigger than page size.
  442. *
  443. * Hole are not considered for allocation to keep things simple.
  444. * Assumption is that there won't be hole (all object on same
  445. * alignment).
  446. */
  447. struct radeon_sa_manager {
  448. wait_queue_head_t wq;
  449. struct radeon_bo *bo;
  450. struct list_head *hole;
  451. struct list_head flist[RADEON_NUM_RINGS];
  452. struct list_head olist;
  453. unsigned size;
  454. uint64_t gpu_addr;
  455. void *cpu_ptr;
  456. uint32_t domain;
  457. uint32_t align;
  458. };
  459. struct radeon_sa_bo;
  460. /* sub-allocation buffer */
  461. struct radeon_sa_bo {
  462. struct list_head olist;
  463. struct list_head flist;
  464. struct radeon_sa_manager *manager;
  465. unsigned soffset;
  466. unsigned eoffset;
  467. struct radeon_fence *fence;
  468. };
  469. /*
  470. * GEM objects.
  471. */
  472. struct radeon_gem {
  473. struct mutex mutex;
  474. struct list_head objects;
  475. };
  476. int radeon_gem_init(struct radeon_device *rdev);
  477. void radeon_gem_fini(struct radeon_device *rdev);
  478. int radeon_gem_object_create(struct radeon_device *rdev, int size,
  479. int alignment, int initial_domain,
  480. bool discardable, bool kernel,
  481. struct drm_gem_object **obj);
  482. int radeon_mode_dumb_create(struct drm_file *file_priv,
  483. struct drm_device *dev,
  484. struct drm_mode_create_dumb *args);
  485. int radeon_mode_dumb_mmap(struct drm_file *filp,
  486. struct drm_device *dev,
  487. uint32_t handle, uint64_t *offset_p);
  488. /*
  489. * Semaphores.
  490. */
  491. /* everything here is constant */
  492. struct radeon_semaphore {
  493. struct radeon_sa_bo *sa_bo;
  494. signed waiters;
  495. uint64_t gpu_addr;
  496. struct radeon_fence *sync_to[RADEON_NUM_RINGS];
  497. };
  498. int radeon_semaphore_create(struct radeon_device *rdev,
  499. struct radeon_semaphore **semaphore);
  500. bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring,
  501. struct radeon_semaphore *semaphore);
  502. bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring,
  503. struct radeon_semaphore *semaphore);
  504. void radeon_semaphore_sync_to(struct radeon_semaphore *semaphore,
  505. struct radeon_fence *fence);
  506. int radeon_semaphore_sync_rings(struct radeon_device *rdev,
  507. struct radeon_semaphore *semaphore,
  508. int waiting_ring);
  509. void radeon_semaphore_free(struct radeon_device *rdev,
  510. struct radeon_semaphore **semaphore,
  511. struct radeon_fence *fence);
  512. /*
  513. * GART structures, functions & helpers
  514. */
  515. struct radeon_mc;
  516. #define RADEON_GPU_PAGE_SIZE 4096
  517. #define RADEON_GPU_PAGE_MASK (RADEON_GPU_PAGE_SIZE - 1)
  518. #define RADEON_GPU_PAGE_SHIFT 12
  519. #define RADEON_GPU_PAGE_ALIGN(a) (((a) + RADEON_GPU_PAGE_MASK) & ~RADEON_GPU_PAGE_MASK)
  520. struct radeon_gart {
  521. dma_addr_t table_addr;
  522. struct radeon_bo *robj;
  523. void *ptr;
  524. unsigned num_gpu_pages;
  525. unsigned num_cpu_pages;
  526. unsigned table_size;
  527. struct page **pages;
  528. dma_addr_t *pages_addr;
  529. bool ready;
  530. };
  531. int radeon_gart_table_ram_alloc(struct radeon_device *rdev);
  532. void radeon_gart_table_ram_free(struct radeon_device *rdev);
  533. int radeon_gart_table_vram_alloc(struct radeon_device *rdev);
  534. void radeon_gart_table_vram_free(struct radeon_device *rdev);
  535. int radeon_gart_table_vram_pin(struct radeon_device *rdev);
  536. void radeon_gart_table_vram_unpin(struct radeon_device *rdev);
  537. int radeon_gart_init(struct radeon_device *rdev);
  538. void radeon_gart_fini(struct radeon_device *rdev);
  539. void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
  540. int pages);
  541. int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
  542. int pages, struct page **pagelist,
  543. dma_addr_t *dma_addr);
  544. void radeon_gart_restore(struct radeon_device *rdev);
  545. /*
  546. * GPU MC structures, functions & helpers
  547. */
  548. struct radeon_mc {
  549. resource_size_t aper_size;
  550. resource_size_t aper_base;
  551. resource_size_t agp_base;
  552. /* for some chips with <= 32MB we need to lie
  553. * about vram size near mc fb location */
  554. u64 mc_vram_size;
  555. u64 visible_vram_size;
  556. u64 gtt_size;
  557. u64 gtt_start;
  558. u64 gtt_end;
  559. u64 vram_start;
  560. u64 vram_end;
  561. unsigned vram_width;
  562. u64 real_vram_size;
  563. int vram_mtrr;
  564. bool vram_is_ddr;
  565. bool igp_sideport_enabled;
  566. u64 gtt_base_align;
  567. u64 mc_mask;
  568. };
  569. bool radeon_combios_sideport_present(struct radeon_device *rdev);
  570. bool radeon_atombios_sideport_present(struct radeon_device *rdev);
  571. /*
  572. * GPU scratch registers structures, functions & helpers
  573. */
  574. struct radeon_scratch {
  575. unsigned num_reg;
  576. uint32_t reg_base;
  577. bool free[32];
  578. uint32_t reg[32];
  579. };
  580. int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg);
  581. void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);
  582. /*
  583. * GPU doorbell structures, functions & helpers
  584. */
  585. #define RADEON_MAX_DOORBELLS 1024 /* Reserve at most 1024 doorbell slots for radeon-owned rings. */
  586. struct radeon_doorbell {
  587. /* doorbell mmio */
  588. resource_size_t base;
  589. resource_size_t size;
  590. u32 __iomem *ptr;
  591. u32 num_doorbells; /* Number of doorbells actually reserved for radeon. */
  592. unsigned long used[DIV_ROUND_UP(RADEON_MAX_DOORBELLS, BITS_PER_LONG)];
  593. };
  594. int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
  595. void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
  596. /*
  597. * IRQS.
  598. */
  599. struct radeon_unpin_work {
  600. struct work_struct work;
  601. struct radeon_device *rdev;
  602. int crtc_id;
  603. struct radeon_fence *fence;
  604. struct drm_pending_vblank_event *event;
  605. struct radeon_bo *old_rbo;
  606. u64 new_crtc_base;
  607. };
  608. struct r500_irq_stat_regs {
  609. u32 disp_int;
  610. u32 hdmi0_status;
  611. };
  612. struct r600_irq_stat_regs {
  613. u32 disp_int;
  614. u32 disp_int_cont;
  615. u32 disp_int_cont2;
  616. u32 d1grph_int;
  617. u32 d2grph_int;
  618. u32 hdmi0_status;
  619. u32 hdmi1_status;
  620. };
  621. struct evergreen_irq_stat_regs {
  622. u32 disp_int;
  623. u32 disp_int_cont;
  624. u32 disp_int_cont2;
  625. u32 disp_int_cont3;
  626. u32 disp_int_cont4;
  627. u32 disp_int_cont5;
  628. u32 d1grph_int;
  629. u32 d2grph_int;
  630. u32 d3grph_int;
  631. u32 d4grph_int;
  632. u32 d5grph_int;
  633. u32 d6grph_int;
  634. u32 afmt_status1;
  635. u32 afmt_status2;
  636. u32 afmt_status3;
  637. u32 afmt_status4;
  638. u32 afmt_status5;
  639. u32 afmt_status6;
  640. };
  641. struct cik_irq_stat_regs {
  642. u32 disp_int;
  643. u32 disp_int_cont;
  644. u32 disp_int_cont2;
  645. u32 disp_int_cont3;
  646. u32 disp_int_cont4;
  647. u32 disp_int_cont5;
  648. u32 disp_int_cont6;
  649. };
  650. union radeon_irq_stat_regs {
  651. struct r500_irq_stat_regs r500;
  652. struct r600_irq_stat_regs r600;
  653. struct evergreen_irq_stat_regs evergreen;
  654. struct cik_irq_stat_regs cik;
  655. };
  656. #define RADEON_MAX_HPD_PINS 6
  657. #define RADEON_MAX_CRTCS 6
  658. #define RADEON_MAX_AFMT_BLOCKS 7
  659. struct radeon_irq {
  660. bool installed;
  661. spinlock_t lock;
  662. atomic_t ring_int[RADEON_NUM_RINGS];
  663. bool crtc_vblank_int[RADEON_MAX_CRTCS];
  664. atomic_t pflip[RADEON_MAX_CRTCS];
  665. wait_queue_head_t vblank_queue;
  666. bool hpd[RADEON_MAX_HPD_PINS];
  667. bool afmt[RADEON_MAX_AFMT_BLOCKS];
  668. union radeon_irq_stat_regs stat_regs;
  669. bool dpm_thermal;
  670. };
  671. int radeon_irq_kms_init(struct radeon_device *rdev);
  672. void radeon_irq_kms_fini(struct radeon_device *rdev);
  673. void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring);
  674. void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring);
  675. void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
  676. void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
  677. void radeon_irq_kms_enable_afmt(struct radeon_device *rdev, int block);
  678. void radeon_irq_kms_disable_afmt(struct radeon_device *rdev, int block);
  679. void radeon_irq_kms_enable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
  680. void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
  681. /*
  682. * CP & rings.
  683. */
  684. struct radeon_ib {
  685. struct radeon_sa_bo *sa_bo;
  686. uint32_t length_dw;
  687. uint64_t gpu_addr;
  688. uint32_t *ptr;
  689. int ring;
  690. struct radeon_fence *fence;
  691. struct radeon_vm *vm;
  692. bool is_const_ib;
  693. struct radeon_semaphore *semaphore;
  694. };
  695. struct radeon_ring {
  696. struct radeon_bo *ring_obj;
  697. volatile uint32_t *ring;
  698. unsigned rptr;
  699. unsigned rptr_offs;
  700. unsigned rptr_reg;
  701. unsigned rptr_save_reg;
  702. u64 next_rptr_gpu_addr;
  703. volatile u32 *next_rptr_cpu_addr;
  704. unsigned wptr;
  705. unsigned wptr_old;
  706. unsigned wptr_reg;
  707. unsigned ring_size;
  708. unsigned ring_free_dw;
  709. int count_dw;
  710. unsigned long last_activity;
  711. unsigned last_rptr;
  712. uint64_t gpu_addr;
  713. uint32_t align_mask;
  714. uint32_t ptr_mask;
  715. bool ready;
  716. u32 nop;
  717. u32 idx;
  718. u64 last_semaphore_signal_addr;
  719. u64 last_semaphore_wait_addr;
  720. /* for CIK queues */
  721. u32 me;
  722. u32 pipe;
  723. u32 queue;
  724. struct radeon_bo *mqd_obj;
  725. u32 doorbell_index;
  726. unsigned wptr_offs;
  727. };
  728. struct radeon_mec {
  729. struct radeon_bo *hpd_eop_obj;
  730. u64 hpd_eop_gpu_addr;
  731. u32 num_pipe;
  732. u32 num_mec;
  733. u32 num_queue;
  734. };
  735. /*
  736. * VM
  737. */
  738. /* maximum number of VMIDs */
  739. #define RADEON_NUM_VM 16
  740. /* defines number of bits in page table versus page directory,
  741. * a page is 4KB so we have 12 bits offset, 9 bits in the page
  742. * table and the remaining 19 bits are in the page directory */
  743. #define RADEON_VM_BLOCK_SIZE 9
  744. /* number of entries in page table */
  745. #define RADEON_VM_PTE_COUNT (1 << RADEON_VM_BLOCK_SIZE)
  746. /* PTBs (Page Table Blocks) need to be aligned to 32K */
  747. #define RADEON_VM_PTB_ALIGN_SIZE 32768
  748. #define RADEON_VM_PTB_ALIGN_MASK (RADEON_VM_PTB_ALIGN_SIZE - 1)
  749. #define RADEON_VM_PTB_ALIGN(a) (((a) + RADEON_VM_PTB_ALIGN_MASK) & ~RADEON_VM_PTB_ALIGN_MASK)
  750. #define R600_PTE_VALID (1 << 0)
  751. #define R600_PTE_SYSTEM (1 << 1)
  752. #define R600_PTE_SNOOPED (1 << 2)
  753. #define R600_PTE_READABLE (1 << 5)
  754. #define R600_PTE_WRITEABLE (1 << 6)
  755. struct radeon_vm {
  756. struct list_head list;
  757. struct list_head va;
  758. unsigned id;
  759. /* contains the page directory */
  760. struct radeon_sa_bo *page_directory;
  761. uint64_t pd_gpu_addr;
  762. /* array of page tables, one for each page directory entry */
  763. struct radeon_sa_bo **page_tables;
  764. struct mutex mutex;
  765. /* last fence for cs using this vm */
  766. struct radeon_fence *fence;
  767. /* last flush or NULL if we still need to flush */
  768. struct radeon_fence *last_flush;
  769. };
  770. struct radeon_vm_manager {
  771. struct mutex lock;
  772. struct list_head lru_vm;
  773. struct radeon_fence *active[RADEON_NUM_VM];
  774. struct radeon_sa_manager sa_manager;
  775. uint32_t max_pfn;
  776. /* number of VMIDs */
  777. unsigned nvm;
  778. /* vram base address for page table entry */
  779. u64 vram_base_offset;
  780. /* is vm enabled? */
  781. bool enabled;
  782. };
  783. /*
  784. * file private structure
  785. */
  786. struct radeon_fpriv {
  787. struct radeon_vm vm;
  788. };
  789. /*
  790. * R6xx+ IH ring
  791. */
  792. struct r600_ih {
  793. struct radeon_bo *ring_obj;
  794. volatile uint32_t *ring;
  795. unsigned rptr;
  796. unsigned ring_size;
  797. uint64_t gpu_addr;
  798. uint32_t ptr_mask;
  799. atomic_t lock;
  800. bool enabled;
  801. };
  802. /*
  803. * RLC stuff
  804. */
  805. #include "clearstate_defs.h"
  806. struct radeon_rlc {
  807. /* for power gating */
  808. struct radeon_bo *save_restore_obj;
  809. uint64_t save_restore_gpu_addr;
  810. volatile uint32_t *sr_ptr;
  811. const u32 *reg_list;
  812. u32 reg_list_size;
  813. /* for clear state */
  814. struct radeon_bo *clear_state_obj;
  815. uint64_t clear_state_gpu_addr;
  816. volatile uint32_t *cs_ptr;
  817. const struct cs_section_def *cs_data;
  818. u32 clear_state_size;
  819. /* for cp tables */
  820. struct radeon_bo *cp_table_obj;
  821. uint64_t cp_table_gpu_addr;
  822. volatile uint32_t *cp_table_ptr;
  823. u32 cp_table_size;
  824. };
  825. int radeon_ib_get(struct radeon_device *rdev, int ring,
  826. struct radeon_ib *ib, struct radeon_vm *vm,
  827. unsigned size);
  828. void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib);
  829. int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
  830. struct radeon_ib *const_ib);
  831. int radeon_ib_pool_init(struct radeon_device *rdev);
  832. void radeon_ib_pool_fini(struct radeon_device *rdev);
  833. int radeon_ib_ring_tests(struct radeon_device *rdev);
  834. /* Ring access between begin & end cannot sleep */
  835. bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
  836. struct radeon_ring *ring);
  837. void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *cp);
  838. int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
  839. int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
  840. void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *cp);
  841. void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *cp);
  842. void radeon_ring_undo(struct radeon_ring *ring);
  843. void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *cp);
  844. int radeon_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
  845. void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring);
  846. void radeon_ring_lockup_update(struct radeon_ring *ring);
  847. bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
  848. unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
  849. uint32_t **data);
  850. int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
  851. unsigned size, uint32_t *data);
  852. int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ring_size,
  853. unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg, u32 nop);
  854. void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *cp);
  855. /* r600 async dma */
  856. void r600_dma_stop(struct radeon_device *rdev);
  857. int r600_dma_resume(struct radeon_device *rdev);
  858. void r600_dma_fini(struct radeon_device *rdev);
  859. void cayman_dma_stop(struct radeon_device *rdev);
  860. int cayman_dma_resume(struct radeon_device *rdev);
  861. void cayman_dma_fini(struct radeon_device *rdev);
  862. /*
  863. * CS.
  864. */
  865. struct radeon_cs_reloc {
  866. struct drm_gem_object *gobj;
  867. struct radeon_bo *robj;
  868. struct radeon_bo_list lobj;
  869. uint32_t handle;
  870. uint32_t flags;
  871. };
  872. struct radeon_cs_chunk {
  873. uint32_t chunk_id;
  874. uint32_t length_dw;
  875. uint32_t *kdata;
  876. void __user *user_ptr;
  877. };
  878. struct radeon_cs_parser {
  879. struct device *dev;
  880. struct radeon_device *rdev;
  881. struct drm_file *filp;
  882. /* chunks */
  883. unsigned nchunks;
  884. struct radeon_cs_chunk *chunks;
  885. uint64_t *chunks_array;
  886. /* IB */
  887. unsigned idx;
  888. /* relocations */
  889. unsigned nrelocs;
  890. struct radeon_cs_reloc *relocs;
  891. struct radeon_cs_reloc **relocs_ptr;
  892. struct list_head validated;
  893. unsigned dma_reloc_idx;
  894. /* indices of various chunks */
  895. int chunk_ib_idx;
  896. int chunk_relocs_idx;
  897. int chunk_flags_idx;
  898. int chunk_const_ib_idx;
  899. struct radeon_ib ib;
  900. struct radeon_ib const_ib;
  901. void *track;
  902. unsigned family;
  903. int parser_error;
  904. u32 cs_flags;
  905. u32 ring;
  906. s32 priority;
  907. struct ww_acquire_ctx ticket;
  908. };
  909. static inline u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
  910. {
  911. struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
  912. if (ibc->kdata)
  913. return ibc->kdata[idx];
  914. return p->ib.ptr[idx];
  915. }
  916. struct radeon_cs_packet {
  917. unsigned idx;
  918. unsigned type;
  919. unsigned reg;
  920. unsigned opcode;
  921. int count;
  922. unsigned one_reg_wr;
  923. };
  924. typedef int (*radeon_packet0_check_t)(struct radeon_cs_parser *p,
  925. struct radeon_cs_packet *pkt,
  926. unsigned idx, unsigned reg);
  927. typedef int (*radeon_packet3_check_t)(struct radeon_cs_parser *p,
  928. struct radeon_cs_packet *pkt);
  929. /*
  930. * AGP
  931. */
  932. int radeon_agp_init(struct radeon_device *rdev);
  933. void radeon_agp_resume(struct radeon_device *rdev);
  934. void radeon_agp_suspend(struct radeon_device *rdev);
  935. void radeon_agp_fini(struct radeon_device *rdev);
  936. /*
  937. * Writeback
  938. */
  939. struct radeon_wb {
  940. struct radeon_bo *wb_obj;
  941. volatile uint32_t *wb;
  942. uint64_t gpu_addr;
  943. bool enabled;
  944. bool use_event;
  945. };
  946. #define RADEON_WB_SCRATCH_OFFSET 0
  947. #define RADEON_WB_RING0_NEXT_RPTR 256
  948. #define RADEON_WB_CP_RPTR_OFFSET 1024
  949. #define RADEON_WB_CP1_RPTR_OFFSET 1280
  950. #define RADEON_WB_CP2_RPTR_OFFSET 1536
  951. #define R600_WB_DMA_RPTR_OFFSET 1792
  952. #define R600_WB_IH_WPTR_OFFSET 2048
  953. #define CAYMAN_WB_DMA1_RPTR_OFFSET 2304
  954. #define R600_WB_EVENT_OFFSET 3072
  955. #define CIK_WB_CP1_WPTR_OFFSET 3328
  956. #define CIK_WB_CP2_WPTR_OFFSET 3584
  957. /**
  958. * struct radeon_pm - power management datas
  959. * @max_bandwidth: maximum bandwidth the gpu has (MByte/s)
  960. * @igp_sideport_mclk: sideport memory clock Mhz (rs690,rs740,rs780,rs880)
  961. * @igp_system_mclk: system clock Mhz (rs690,rs740,rs780,rs880)
  962. * @igp_ht_link_clk: ht link clock Mhz (rs690,rs740,rs780,rs880)
  963. * @igp_ht_link_width: ht link width in bits (rs690,rs740,rs780,rs880)
  964. * @k8_bandwidth: k8 bandwidth the gpu has (MByte/s) (IGP)
  965. * @sideport_bandwidth: sideport bandwidth the gpu has (MByte/s) (IGP)
  966. * @ht_bandwidth: ht bandwidth the gpu has (MByte/s) (IGP)
  967. * @core_bandwidth: core GPU bandwidth the gpu has (MByte/s) (IGP)
  968. * @sclk: GPU clock Mhz (core bandwidth depends of this clock)
  969. * @needed_bandwidth: current bandwidth needs
  970. *
  971. * It keeps track of various data needed to take powermanagement decision.
  972. * Bandwidth need is used to determine minimun clock of the GPU and memory.
  973. * Equation between gpu/memory clock and available bandwidth is hw dependent
  974. * (type of memory, bus size, efficiency, ...)
  975. */
  976. enum radeon_pm_method {
  977. PM_METHOD_PROFILE,
  978. PM_METHOD_DYNPM,
  979. PM_METHOD_DPM,
  980. };
  981. enum radeon_dynpm_state {
  982. DYNPM_STATE_DISABLED,
  983. DYNPM_STATE_MINIMUM,
  984. DYNPM_STATE_PAUSED,
  985. DYNPM_STATE_ACTIVE,
  986. DYNPM_STATE_SUSPENDED,
  987. };
  988. enum radeon_dynpm_action {
  989. DYNPM_ACTION_NONE,
  990. DYNPM_ACTION_MINIMUM,
  991. DYNPM_ACTION_DOWNCLOCK,
  992. DYNPM_ACTION_UPCLOCK,
  993. DYNPM_ACTION_DEFAULT
  994. };
  995. enum radeon_voltage_type {
  996. VOLTAGE_NONE = 0,
  997. VOLTAGE_GPIO,
  998. VOLTAGE_VDDC,
  999. VOLTAGE_SW
  1000. };
  1001. enum radeon_pm_state_type {
  1002. /* not used for dpm */
  1003. POWER_STATE_TYPE_DEFAULT,
  1004. POWER_STATE_TYPE_POWERSAVE,
  1005. /* user selectable states */
  1006. POWER_STATE_TYPE_BATTERY,
  1007. POWER_STATE_TYPE_BALANCED,
  1008. POWER_STATE_TYPE_PERFORMANCE,
  1009. /* internal states */
  1010. POWER_STATE_TYPE_INTERNAL_UVD,
  1011. POWER_STATE_TYPE_INTERNAL_UVD_SD,
  1012. POWER_STATE_TYPE_INTERNAL_UVD_HD,
  1013. POWER_STATE_TYPE_INTERNAL_UVD_HD2,
  1014. POWER_STATE_TYPE_INTERNAL_UVD_MVC,
  1015. POWER_STATE_TYPE_INTERNAL_BOOT,
  1016. POWER_STATE_TYPE_INTERNAL_THERMAL,
  1017. POWER_STATE_TYPE_INTERNAL_ACPI,
  1018. POWER_STATE_TYPE_INTERNAL_ULV,
  1019. POWER_STATE_TYPE_INTERNAL_3DPERF,
  1020. };
  1021. enum radeon_pm_profile_type {
  1022. PM_PROFILE_DEFAULT,
  1023. PM_PROFILE_AUTO,
  1024. PM_PROFILE_LOW,
  1025. PM_PROFILE_MID,
  1026. PM_PROFILE_HIGH,
  1027. };
  1028. #define PM_PROFILE_DEFAULT_IDX 0
  1029. #define PM_PROFILE_LOW_SH_IDX 1
  1030. #define PM_PROFILE_MID_SH_IDX 2
  1031. #define PM_PROFILE_HIGH_SH_IDX 3
  1032. #define PM_PROFILE_LOW_MH_IDX 4
  1033. #define PM_PROFILE_MID_MH_IDX 5
  1034. #define PM_PROFILE_HIGH_MH_IDX 6
  1035. #define PM_PROFILE_MAX 7
  1036. struct radeon_pm_profile {
  1037. int dpms_off_ps_idx;
  1038. int dpms_on_ps_idx;
  1039. int dpms_off_cm_idx;
  1040. int dpms_on_cm_idx;
  1041. };
  1042. enum radeon_int_thermal_type {
  1043. THERMAL_TYPE_NONE,
  1044. THERMAL_TYPE_EXTERNAL,
  1045. THERMAL_TYPE_EXTERNAL_GPIO,
  1046. THERMAL_TYPE_RV6XX,
  1047. THERMAL_TYPE_RV770,
  1048. THERMAL_TYPE_ADT7473_WITH_INTERNAL,
  1049. THERMAL_TYPE_EVERGREEN,
  1050. THERMAL_TYPE_SUMO,
  1051. THERMAL_TYPE_NI,
  1052. THERMAL_TYPE_SI,
  1053. THERMAL_TYPE_EMC2103_WITH_INTERNAL,
  1054. THERMAL_TYPE_CI,
  1055. THERMAL_TYPE_KV,
  1056. };
  1057. struct radeon_voltage {
  1058. enum radeon_voltage_type type;
  1059. /* gpio voltage */
  1060. struct radeon_gpio_rec gpio;
  1061. u32 delay; /* delay in usec from voltage drop to sclk change */
  1062. bool active_high; /* voltage drop is active when bit is high */
  1063. /* VDDC voltage */
  1064. u8 vddc_id; /* index into vddc voltage table */
  1065. u8 vddci_id; /* index into vddci voltage table */
  1066. bool vddci_enabled;
  1067. /* r6xx+ sw */
  1068. u16 voltage;
  1069. /* evergreen+ vddci */
  1070. u16 vddci;
  1071. };
  1072. /* clock mode flags */
  1073. #define RADEON_PM_MODE_NO_DISPLAY (1 << 0)
  1074. struct radeon_pm_clock_info {
  1075. /* memory clock */
  1076. u32 mclk;
  1077. /* engine clock */
  1078. u32 sclk;
  1079. /* voltage info */
  1080. struct radeon_voltage voltage;
  1081. /* standardized clock flags */
  1082. u32 flags;
  1083. };
  1084. /* state flags */
  1085. #define RADEON_PM_STATE_SINGLE_DISPLAY_ONLY (1 << 0)
  1086. struct radeon_power_state {
  1087. enum radeon_pm_state_type type;
  1088. struct radeon_pm_clock_info *clock_info;
  1089. /* number of valid clock modes in this power state */
  1090. int num_clock_modes;
  1091. struct radeon_pm_clock_info *default_clock_mode;
  1092. /* standardized state flags */
  1093. u32 flags;
  1094. u32 misc; /* vbios specific flags */
  1095. u32 misc2; /* vbios specific flags */
  1096. int pcie_lanes; /* pcie lanes */
  1097. };
  1098. /*
  1099. * Some modes are overclocked by very low value, accept them
  1100. */
  1101. #define RADEON_MODE_OVERCLOCK_MARGIN 500 /* 5 MHz */
  1102. enum radeon_dpm_auto_throttle_src {
  1103. RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL,
  1104. RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL
  1105. };
  1106. enum radeon_dpm_event_src {
  1107. RADEON_DPM_EVENT_SRC_ANALOG = 0,
  1108. RADEON_DPM_EVENT_SRC_EXTERNAL = 1,
  1109. RADEON_DPM_EVENT_SRC_DIGITAL = 2,
  1110. RADEON_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
  1111. RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
  1112. };
  1113. struct radeon_ps {
  1114. u32 caps; /* vbios flags */
  1115. u32 class; /* vbios flags */
  1116. u32 class2; /* vbios flags */
  1117. /* UVD clocks */
  1118. u32 vclk;
  1119. u32 dclk;
  1120. /* VCE clocks */
  1121. u32 evclk;
  1122. u32 ecclk;
  1123. /* asic priv */
  1124. void *ps_priv;
  1125. };
  1126. struct radeon_dpm_thermal {
  1127. /* thermal interrupt work */
  1128. struct work_struct work;
  1129. /* low temperature threshold */
  1130. int min_temp;
  1131. /* high temperature threshold */
  1132. int max_temp;
  1133. /* was interrupt low to high or high to low */
  1134. bool high_to_low;
  1135. };
  1136. enum radeon_clk_action
  1137. {
  1138. RADEON_SCLK_UP = 1,
  1139. RADEON_SCLK_DOWN
  1140. };
  1141. struct radeon_blacklist_clocks
  1142. {
  1143. u32 sclk;
  1144. u32 mclk;
  1145. enum radeon_clk_action action;
  1146. };
  1147. struct radeon_clock_and_voltage_limits {
  1148. u32 sclk;
  1149. u32 mclk;
  1150. u16 vddc;
  1151. u16 vddci;
  1152. };
  1153. struct radeon_clock_array {
  1154. u32 count;
  1155. u32 *values;
  1156. };
  1157. struct radeon_clock_voltage_dependency_entry {
  1158. u32 clk;
  1159. u16 v;
  1160. };
  1161. struct radeon_clock_voltage_dependency_table {
  1162. u32 count;
  1163. struct radeon_clock_voltage_dependency_entry *entries;
  1164. };
  1165. union radeon_cac_leakage_entry {
  1166. struct {
  1167. u16 vddc;
  1168. u32 leakage;
  1169. };
  1170. struct {
  1171. u16 vddc1;
  1172. u16 vddc2;
  1173. u16 vddc3;
  1174. };
  1175. };
  1176. struct radeon_cac_leakage_table {
  1177. u32 count;
  1178. union radeon_cac_leakage_entry *entries;
  1179. };
  1180. struct radeon_phase_shedding_limits_entry {
  1181. u16 voltage;
  1182. u32 sclk;
  1183. u32 mclk;
  1184. };
  1185. struct radeon_phase_shedding_limits_table {
  1186. u32 count;
  1187. struct radeon_phase_shedding_limits_entry *entries;
  1188. };
  1189. struct radeon_uvd_clock_voltage_dependency_entry {
  1190. u32 vclk;
  1191. u32 dclk;
  1192. u16 v;
  1193. };
  1194. struct radeon_uvd_clock_voltage_dependency_table {
  1195. u8 count;
  1196. struct radeon_uvd_clock_voltage_dependency_entry *entries;
  1197. };
  1198. struct radeon_vce_clock_voltage_dependency_entry {
  1199. u32 ecclk;
  1200. u32 evclk;
  1201. u16 v;
  1202. };
  1203. struct radeon_vce_clock_voltage_dependency_table {
  1204. u8 count;
  1205. struct radeon_vce_clock_voltage_dependency_entry *entries;
  1206. };
  1207. struct radeon_ppm_table {
  1208. u8 ppm_design;
  1209. u16 cpu_core_number;
  1210. u32 platform_tdp;
  1211. u32 small_ac_platform_tdp;
  1212. u32 platform_tdc;
  1213. u32 small_ac_platform_tdc;
  1214. u32 apu_tdp;
  1215. u32 dgpu_tdp;
  1216. u32 dgpu_ulv_power;
  1217. u32 tj_max;
  1218. };
  1219. struct radeon_cac_tdp_table {
  1220. u16 tdp;
  1221. u16 configurable_tdp;
  1222. u16 tdc;
  1223. u16 battery_power_limit;
  1224. u16 small_power_limit;
  1225. u16 low_cac_leakage;
  1226. u16 high_cac_leakage;
  1227. u16 maximum_power_delivery_limit;
  1228. };
  1229. struct radeon_dpm_dynamic_state {
  1230. struct radeon_clock_voltage_dependency_table vddc_dependency_on_sclk;
  1231. struct radeon_clock_voltage_dependency_table vddci_dependency_on_mclk;
  1232. struct radeon_clock_voltage_dependency_table vddc_dependency_on_mclk;
  1233. struct radeon_clock_voltage_dependency_table mvdd_dependency_on_mclk;
  1234. struct radeon_clock_voltage_dependency_table vddc_dependency_on_dispclk;
  1235. struct radeon_uvd_clock_voltage_dependency_table uvd_clock_voltage_dependency_table;
  1236. struct radeon_vce_clock_voltage_dependency_table vce_clock_voltage_dependency_table;
  1237. struct radeon_clock_voltage_dependency_table samu_clock_voltage_dependency_table;
  1238. struct radeon_clock_voltage_dependency_table acp_clock_voltage_dependency_table;
  1239. struct radeon_clock_array valid_sclk_values;
  1240. struct radeon_clock_array valid_mclk_values;
  1241. struct radeon_clock_and_voltage_limits max_clock_voltage_on_dc;
  1242. struct radeon_clock_and_voltage_limits max_clock_voltage_on_ac;
  1243. u32 mclk_sclk_ratio;
  1244. u32 sclk_mclk_delta;
  1245. u16 vddc_vddci_delta;
  1246. u16 min_vddc_for_pcie_gen2;
  1247. struct radeon_cac_leakage_table cac_leakage_table;
  1248. struct radeon_phase_shedding_limits_table phase_shedding_limits_table;
  1249. struct radeon_ppm_table *ppm_table;
  1250. struct radeon_cac_tdp_table *cac_tdp_table;
  1251. };
  1252. struct radeon_dpm_fan {
  1253. u16 t_min;
  1254. u16 t_med;
  1255. u16 t_high;
  1256. u16 pwm_min;
  1257. u16 pwm_med;
  1258. u16 pwm_high;
  1259. u8 t_hyst;
  1260. u32 cycle_delay;
  1261. u16 t_max;
  1262. bool ucode_fan_control;
  1263. };
  1264. enum radeon_pcie_gen {
  1265. RADEON_PCIE_GEN1 = 0,
  1266. RADEON_PCIE_GEN2 = 1,
  1267. RADEON_PCIE_GEN3 = 2,
  1268. RADEON_PCIE_GEN_INVALID = 0xffff
  1269. };
  1270. enum radeon_dpm_forced_level {
  1271. RADEON_DPM_FORCED_LEVEL_AUTO = 0,
  1272. RADEON_DPM_FORCED_LEVEL_LOW = 1,
  1273. RADEON_DPM_FORCED_LEVEL_HIGH = 2,
  1274. };
  1275. struct radeon_dpm {
  1276. struct radeon_ps *ps;
  1277. /* number of valid power states */
  1278. int num_ps;
  1279. /* current power state that is active */
  1280. struct radeon_ps *current_ps;
  1281. /* requested power state */
  1282. struct radeon_ps *requested_ps;
  1283. /* boot up power state */
  1284. struct radeon_ps *boot_ps;
  1285. /* default uvd power state */
  1286. struct radeon_ps *uvd_ps;
  1287. enum radeon_pm_state_type state;
  1288. enum radeon_pm_state_type user_state;
  1289. u32 platform_caps;
  1290. u32 voltage_response_time;
  1291. u32 backbias_response_time;
  1292. void *priv;
  1293. u32 new_active_crtcs;
  1294. int new_active_crtc_count;
  1295. u32 current_active_crtcs;
  1296. int current_active_crtc_count;
  1297. struct radeon_dpm_dynamic_state dyn_state;
  1298. struct radeon_dpm_fan fan;
  1299. u32 tdp_limit;
  1300. u32 near_tdp_limit;
  1301. u32 near_tdp_limit_adjusted;
  1302. u32 sq_ramping_threshold;
  1303. u32 cac_leakage;
  1304. u16 tdp_od_limit;
  1305. u32 tdp_adjustment;
  1306. u16 load_line_slope;
  1307. bool power_control;
  1308. bool ac_power;
  1309. /* special states active */
  1310. bool thermal_active;
  1311. bool uvd_active;
  1312. /* thermal handling */
  1313. struct radeon_dpm_thermal thermal;
  1314. /* forced levels */
  1315. enum radeon_dpm_forced_level forced_level;
  1316. /* track UVD streams */
  1317. unsigned sd;
  1318. unsigned hd;
  1319. };
  1320. void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable);
  1321. struct radeon_pm {
  1322. struct mutex mutex;
  1323. /* write locked while reprogramming mclk */
  1324. struct rw_semaphore mclk_lock;
  1325. u32 active_crtcs;
  1326. int active_crtc_count;
  1327. int req_vblank;
  1328. bool vblank_sync;
  1329. fixed20_12 max_bandwidth;
  1330. fixed20_12 igp_sideport_mclk;
  1331. fixed20_12 igp_system_mclk;
  1332. fixed20_12 igp_ht_link_clk;
  1333. fixed20_12 igp_ht_link_width;
  1334. fixed20_12 k8_bandwidth;
  1335. fixed20_12 sideport_bandwidth;
  1336. fixed20_12 ht_bandwidth;
  1337. fixed20_12 core_bandwidth;
  1338. fixed20_12 sclk;
  1339. fixed20_12 mclk;
  1340. fixed20_12 needed_bandwidth;
  1341. struct radeon_power_state *power_state;
  1342. /* number of valid power states */
  1343. int num_power_states;
  1344. int current_power_state_index;
  1345. int current_clock_mode_index;
  1346. int requested_power_state_index;
  1347. int requested_clock_mode_index;
  1348. int default_power_state_index;
  1349. u32 current_sclk;
  1350. u32 current_mclk;
  1351. u16 current_vddc;
  1352. u16 current_vddci;
  1353. u32 default_sclk;
  1354. u32 default_mclk;
  1355. u16 default_vddc;
  1356. u16 default_vddci;
  1357. struct radeon_i2c_chan *i2c_bus;
  1358. /* selected pm method */
  1359. enum radeon_pm_method pm_method;
  1360. /* dynpm power management */
  1361. struct delayed_work dynpm_idle_work;
  1362. enum radeon_dynpm_state dynpm_state;
  1363. enum radeon_dynpm_action dynpm_planned_action;
  1364. unsigned long dynpm_action_timeout;
  1365. bool dynpm_can_upclock;
  1366. bool dynpm_can_downclock;
  1367. /* profile-based power management */
  1368. enum radeon_pm_profile_type profile;
  1369. int profile_index;
  1370. struct radeon_pm_profile profiles[PM_PROFILE_MAX];
  1371. /* internal thermal controller on rv6xx+ */
  1372. enum radeon_int_thermal_type int_thermal_type;
  1373. struct device *int_hwmon_dev;
  1374. /* dpm */
  1375. bool dpm_enabled;
  1376. struct radeon_dpm dpm;
  1377. };
  1378. int radeon_pm_get_type_index(struct radeon_device *rdev,
  1379. enum radeon_pm_state_type ps_type,
  1380. int instance);
  1381. /*
  1382. * UVD
  1383. */
  1384. #define RADEON_MAX_UVD_HANDLES 10
  1385. #define RADEON_UVD_STACK_SIZE (1024*1024)
  1386. #define RADEON_UVD_HEAP_SIZE (1024*1024)
  1387. struct radeon_uvd {
  1388. struct radeon_bo *vcpu_bo;
  1389. void *cpu_addr;
  1390. uint64_t gpu_addr;
  1391. void *saved_bo;
  1392. atomic_t handles[RADEON_MAX_UVD_HANDLES];
  1393. struct drm_file *filp[RADEON_MAX_UVD_HANDLES];
  1394. unsigned img_size[RADEON_MAX_UVD_HANDLES];
  1395. struct delayed_work idle_work;
  1396. };
  1397. int radeon_uvd_init(struct radeon_device *rdev);
  1398. void radeon_uvd_fini(struct radeon_device *rdev);
  1399. int radeon_uvd_suspend(struct radeon_device *rdev);
  1400. int radeon_uvd_resume(struct radeon_device *rdev);
  1401. int radeon_uvd_get_create_msg(struct radeon_device *rdev, int ring,
  1402. uint32_t handle, struct radeon_fence **fence);
  1403. int radeon_uvd_get_destroy_msg(struct radeon_device *rdev, int ring,
  1404. uint32_t handle, struct radeon_fence **fence);
  1405. void radeon_uvd_force_into_uvd_segment(struct radeon_bo *rbo);
  1406. void radeon_uvd_free_handles(struct radeon_device *rdev,
  1407. struct drm_file *filp);
  1408. int radeon_uvd_cs_parse(struct radeon_cs_parser *parser);
  1409. void radeon_uvd_note_usage(struct radeon_device *rdev);
  1410. int radeon_uvd_calc_upll_dividers(struct radeon_device *rdev,
  1411. unsigned vclk, unsigned dclk,
  1412. unsigned vco_min, unsigned vco_max,
  1413. unsigned fb_factor, unsigned fb_mask,
  1414. unsigned pd_min, unsigned pd_max,
  1415. unsigned pd_even,
  1416. unsigned *optimal_fb_div,
  1417. unsigned *optimal_vclk_div,
  1418. unsigned *optimal_dclk_div);
  1419. int radeon_uvd_send_upll_ctlreq(struct radeon_device *rdev,
  1420. unsigned cg_upll_func_cntl);
  1421. struct r600_audio_pin {
  1422. int channels;
  1423. int rate;
  1424. int bits_per_sample;
  1425. u8 status_bits;
  1426. u8 category_code;
  1427. u32 offset;
  1428. bool connected;
  1429. u32 id;
  1430. };
  1431. struct r600_audio {
  1432. bool enabled;
  1433. struct r600_audio_pin pin[RADEON_MAX_AFMT_BLOCKS];
  1434. int num_pins;
  1435. };
  1436. /*
  1437. * Benchmarking
  1438. */
  1439. void radeon_benchmark(struct radeon_device *rdev, int test_number);
  1440. /*
  1441. * Testing
  1442. */
  1443. void radeon_test_moves(struct radeon_device *rdev);
  1444. void radeon_test_ring_sync(struct radeon_device *rdev,
  1445. struct radeon_ring *cpA,
  1446. struct radeon_ring *cpB);
  1447. void radeon_test_syncing(struct radeon_device *rdev);
  1448. /*
  1449. * Debugfs
  1450. */
  1451. struct radeon_debugfs {
  1452. struct drm_info_list *files;
  1453. unsigned num_files;
  1454. };
  1455. int radeon_debugfs_add_files(struct radeon_device *rdev,
  1456. struct drm_info_list *files,
  1457. unsigned nfiles);
  1458. int radeon_debugfs_fence_init(struct radeon_device *rdev);
  1459. /*
  1460. * ASIC ring specific functions.
  1461. */
  1462. struct radeon_asic_ring {
  1463. /* ring read/write ptr handling */
  1464. u32 (*get_rptr)(struct radeon_device *rdev, struct radeon_ring *ring);
  1465. u32 (*get_wptr)(struct radeon_device *rdev, struct radeon_ring *ring);
  1466. void (*set_wptr)(struct radeon_device *rdev, struct radeon_ring *ring);
  1467. /* validating and patching of IBs */
  1468. int (*ib_parse)(struct radeon_device *rdev, struct radeon_ib *ib);
  1469. int (*cs_parse)(struct radeon_cs_parser *p);
  1470. /* command emmit functions */
  1471. void (*ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib);
  1472. void (*emit_fence)(struct radeon_device *rdev, struct radeon_fence *fence);
  1473. bool (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp,
  1474. struct radeon_semaphore *semaphore, bool emit_wait);
  1475. void (*vm_flush)(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
  1476. /* testing functions */
  1477. int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp);
  1478. int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp);
  1479. bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp);
  1480. /* deprecated */
  1481. void (*ring_start)(struct radeon_device *rdev, struct radeon_ring *cp);
  1482. };
  1483. /*
  1484. * ASIC specific functions.
  1485. */
  1486. struct radeon_asic {
  1487. int (*init)(struct radeon_device *rdev);
  1488. void (*fini)(struct radeon_device *rdev);
  1489. int (*resume)(struct radeon_device *rdev);
  1490. int (*suspend)(struct radeon_device *rdev);
  1491. void (*vga_set_state)(struct radeon_device *rdev, bool state);
  1492. int (*asic_reset)(struct radeon_device *rdev);
  1493. /* ioctl hw specific callback. Some hw might want to perform special
  1494. * operation on specific ioctl. For instance on wait idle some hw
  1495. * might want to perform and HDP flush through MMIO as it seems that
  1496. * some R6XX/R7XX hw doesn't take HDP flush into account if programmed
  1497. * through ring.
  1498. */
  1499. void (*ioctl_wait_idle)(struct radeon_device *rdev, struct radeon_bo *bo);
  1500. /* check if 3D engine is idle */
  1501. bool (*gui_idle)(struct radeon_device *rdev);
  1502. /* wait for mc_idle */
  1503. int (*mc_wait_for_idle)(struct radeon_device *rdev);
  1504. /* get the reference clock */
  1505. u32 (*get_xclk)(struct radeon_device *rdev);
  1506. /* get the gpu clock counter */
  1507. uint64_t (*get_gpu_clock_counter)(struct radeon_device *rdev);
  1508. /* gart */
  1509. struct {
  1510. void (*tlb_flush)(struct radeon_device *rdev);
  1511. int (*set_page)(struct radeon_device *rdev, int i, uint64_t addr);
  1512. } gart;
  1513. struct {
  1514. int (*init)(struct radeon_device *rdev);
  1515. void (*fini)(struct radeon_device *rdev);
  1516. void (*set_page)(struct radeon_device *rdev,
  1517. struct radeon_ib *ib,
  1518. uint64_t pe,
  1519. uint64_t addr, unsigned count,
  1520. uint32_t incr, uint32_t flags);
  1521. } vm;
  1522. /* ring specific callbacks */
  1523. struct radeon_asic_ring *ring[RADEON_NUM_RINGS];
  1524. /* irqs */
  1525. struct {
  1526. int (*set)(struct radeon_device *rdev);
  1527. int (*process)(struct radeon_device *rdev);
  1528. } irq;
  1529. /* displays */
  1530. struct {
  1531. /* display watermarks */
  1532. void (*bandwidth_update)(struct radeon_device *rdev);
  1533. /* get frame count */
  1534. u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
  1535. /* wait for vblank */
  1536. void (*wait_for_vblank)(struct radeon_device *rdev, int crtc);
  1537. /* set backlight level */
  1538. void (*set_backlight_level)(struct radeon_encoder *radeon_encoder, u8 level);
  1539. /* get backlight level */
  1540. u8 (*get_backlight_level)(struct radeon_encoder *radeon_encoder);
  1541. /* audio callbacks */
  1542. void (*hdmi_enable)(struct drm_encoder *encoder, bool enable);
  1543. void (*hdmi_setmode)(struct drm_encoder *encoder, struct drm_display_mode *mode);
  1544. } display;
  1545. /* copy functions for bo handling */
  1546. struct {
  1547. int (*blit)(struct radeon_device *rdev,
  1548. uint64_t src_offset,
  1549. uint64_t dst_offset,
  1550. unsigned num_gpu_pages,
  1551. struct radeon_fence **fence);
  1552. u32 blit_ring_index;
  1553. int (*dma)(struct radeon_device *rdev,
  1554. uint64_t src_offset,
  1555. uint64_t dst_offset,
  1556. unsigned num_gpu_pages,
  1557. struct radeon_fence **fence);
  1558. u32 dma_ring_index;
  1559. /* method used for bo copy */
  1560. int (*copy)(struct radeon_device *rdev,
  1561. uint64_t src_offset,
  1562. uint64_t dst_offset,
  1563. unsigned num_gpu_pages,
  1564. struct radeon_fence **fence);
  1565. /* ring used for bo copies */
  1566. u32 copy_ring_index;
  1567. } copy;
  1568. /* surfaces */
  1569. struct {
  1570. int (*set_reg)(struct radeon_device *rdev, int reg,
  1571. uint32_t tiling_flags, uint32_t pitch,
  1572. uint32_t offset, uint32_t obj_size);
  1573. void (*clear_reg)(struct radeon_device *rdev, int reg);
  1574. } surface;
  1575. /* hotplug detect */
  1576. struct {
  1577. void (*init)(struct radeon_device *rdev);
  1578. void (*fini)(struct radeon_device *rdev);
  1579. bool (*sense)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
  1580. void (*set_polarity)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
  1581. } hpd;
  1582. /* static power management */
  1583. struct {
  1584. void (*misc)(struct radeon_device *rdev);
  1585. void (*prepare)(struct radeon_device *rdev);
  1586. void (*finish)(struct radeon_device *rdev);
  1587. void (*init_profile)(struct radeon_device *rdev);
  1588. void (*get_dynpm_state)(struct radeon_device *rdev);
  1589. uint32_t (*get_engine_clock)(struct radeon_device *rdev);
  1590. void (*set_engine_clock)(struct radeon_device *rdev, uint32_t eng_clock);
  1591. uint32_t (*get_memory_clock)(struct radeon_device *rdev);
  1592. void (*set_memory_clock)(struct radeon_device *rdev, uint32_t mem_clock);
  1593. int (*get_pcie_lanes)(struct radeon_device *rdev);
  1594. void (*set_pcie_lanes)(struct radeon_device *rdev, int lanes);
  1595. void (*set_clock_gating)(struct radeon_device *rdev, int enable);
  1596. int (*set_uvd_clocks)(struct radeon_device *rdev, u32 vclk, u32 dclk);
  1597. int (*get_temperature)(struct radeon_device *rdev);
  1598. } pm;
  1599. /* dynamic power management */
  1600. struct {
  1601. int (*init)(struct radeon_device *rdev);
  1602. void (*setup_asic)(struct radeon_device *rdev);
  1603. int (*enable)(struct radeon_device *rdev);
  1604. void (*disable)(struct radeon_device *rdev);
  1605. int (*pre_set_power_state)(struct radeon_device *rdev);
  1606. int (*set_power_state)(struct radeon_device *rdev);
  1607. void (*post_set_power_state)(struct radeon_device *rdev);
  1608. void (*display_configuration_changed)(struct radeon_device *rdev);
  1609. void (*fini)(struct radeon_device *rdev);
  1610. u32 (*get_sclk)(struct radeon_device *rdev, bool low);
  1611. u32 (*get_mclk)(struct radeon_device *rdev, bool low);
  1612. void (*print_power_state)(struct radeon_device *rdev, struct radeon_ps *ps);
  1613. void (*debugfs_print_current_performance_level)(struct radeon_device *rdev, struct seq_file *m);
  1614. int (*force_performance_level)(struct radeon_device *rdev, enum radeon_dpm_forced_level level);
  1615. bool (*vblank_too_short)(struct radeon_device *rdev);
  1616. void (*powergate_uvd)(struct radeon_device *rdev, bool gate);
  1617. void (*enable_bapm)(struct radeon_device *rdev, bool enable);
  1618. } dpm;
  1619. /* pageflipping */
  1620. struct {
  1621. void (*pre_page_flip)(struct radeon_device *rdev, int crtc);
  1622. u32 (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base);
  1623. void (*post_page_flip)(struct radeon_device *rdev, int crtc);
  1624. } pflip;
  1625. };
  1626. /*
  1627. * Asic structures
  1628. */
  1629. struct r100_asic {
  1630. const unsigned *reg_safe_bm;
  1631. unsigned reg_safe_bm_size;
  1632. u32 hdp_cntl;
  1633. };
  1634. struct r300_asic {
  1635. const unsigned *reg_safe_bm;
  1636. unsigned reg_safe_bm_size;
  1637. u32 resync_scratch;
  1638. u32 hdp_cntl;
  1639. };
  1640. struct r600_asic {
  1641. unsigned max_pipes;
  1642. unsigned max_tile_pipes;
  1643. unsigned max_simds;
  1644. unsigned max_backends;
  1645. unsigned max_gprs;
  1646. unsigned max_threads;
  1647. unsigned max_stack_entries;
  1648. unsigned max_hw_contexts;
  1649. unsigned max_gs_threads;
  1650. unsigned sx_max_export_size;
  1651. unsigned sx_max_export_pos_size;
  1652. unsigned sx_max_export_smx_size;
  1653. unsigned sq_num_cf_insts;
  1654. unsigned tiling_nbanks;
  1655. unsigned tiling_npipes;
  1656. unsigned tiling_group_size;
  1657. unsigned tile_config;
  1658. unsigned backend_map;
  1659. };
  1660. struct rv770_asic {
  1661. unsigned max_pipes;
  1662. unsigned max_tile_pipes;
  1663. unsigned max_simds;
  1664. unsigned max_backends;
  1665. unsigned max_gprs;
  1666. unsigned max_threads;
  1667. unsigned max_stack_entries;
  1668. unsigned max_hw_contexts;
  1669. unsigned max_gs_threads;
  1670. unsigned sx_max_export_size;
  1671. unsigned sx_max_export_pos_size;
  1672. unsigned sx_max_export_smx_size;
  1673. unsigned sq_num_cf_insts;
  1674. unsigned sx_num_of_sets;
  1675. unsigned sc_prim_fifo_size;
  1676. unsigned sc_hiz_tile_fifo_size;
  1677. unsigned sc_earlyz_tile_fifo_fize;
  1678. unsigned tiling_nbanks;
  1679. unsigned tiling_npipes;
  1680. unsigned tiling_group_size;
  1681. unsigned tile_config;
  1682. unsigned backend_map;
  1683. };
  1684. struct evergreen_asic {
  1685. unsigned num_ses;
  1686. unsigned max_pipes;
  1687. unsigned max_tile_pipes;
  1688. unsigned max_simds;
  1689. unsigned max_backends;
  1690. unsigned max_gprs;
  1691. unsigned max_threads;
  1692. unsigned max_stack_entries;
  1693. unsigned max_hw_contexts;
  1694. unsigned max_gs_threads;
  1695. unsigned sx_max_export_size;
  1696. unsigned sx_max_export_pos_size;
  1697. unsigned sx_max_export_smx_size;
  1698. unsigned sq_num_cf_insts;
  1699. unsigned sx_num_of_sets;
  1700. unsigned sc_prim_fifo_size;
  1701. unsigned sc_hiz_tile_fifo_size;
  1702. unsigned sc_earlyz_tile_fifo_size;
  1703. unsigned tiling_nbanks;
  1704. unsigned tiling_npipes;
  1705. unsigned tiling_group_size;
  1706. unsigned tile_config;
  1707. unsigned backend_map;
  1708. };
  1709. struct cayman_asic {
  1710. unsigned max_shader_engines;
  1711. unsigned max_pipes_per_simd;
  1712. unsigned max_tile_pipes;
  1713. unsigned max_simds_per_se;
  1714. unsigned max_backends_per_se;
  1715. unsigned max_texture_channel_caches;
  1716. unsigned max_gprs;
  1717. unsigned max_threads;
  1718. unsigned max_gs_threads;
  1719. unsigned max_stack_entries;
  1720. unsigned sx_num_of_sets;
  1721. unsigned sx_max_export_size;
  1722. unsigned sx_max_export_pos_size;
  1723. unsigned sx_max_export_smx_size;
  1724. unsigned max_hw_contexts;
  1725. unsigned sq_num_cf_insts;
  1726. unsigned sc_prim_fifo_size;
  1727. unsigned sc_hiz_tile_fifo_size;
  1728. unsigned sc_earlyz_tile_fifo_size;
  1729. unsigned num_shader_engines;
  1730. unsigned num_shader_pipes_per_simd;
  1731. unsigned num_tile_pipes;
  1732. unsigned num_simds_per_se;
  1733. unsigned num_backends_per_se;
  1734. unsigned backend_disable_mask_per_asic;
  1735. unsigned backend_map;
  1736. unsigned num_texture_channel_caches;
  1737. unsigned mem_max_burst_length_bytes;
  1738. unsigned mem_row_size_in_kb;
  1739. unsigned shader_engine_tile_size;
  1740. unsigned num_gpus;
  1741. unsigned multi_gpu_tile_size;
  1742. unsigned tile_config;
  1743. };
  1744. struct si_asic {
  1745. unsigned max_shader_engines;
  1746. unsigned max_tile_pipes;
  1747. unsigned max_cu_per_sh;
  1748. unsigned max_sh_per_se;
  1749. unsigned max_backends_per_se;
  1750. unsigned max_texture_channel_caches;
  1751. unsigned max_gprs;
  1752. unsigned max_gs_threads;
  1753. unsigned max_hw_contexts;
  1754. unsigned sc_prim_fifo_size_frontend;
  1755. unsigned sc_prim_fifo_size_backend;
  1756. unsigned sc_hiz_tile_fifo_size;
  1757. unsigned sc_earlyz_tile_fifo_size;
  1758. unsigned num_tile_pipes;
  1759. unsigned num_backends_per_se;
  1760. unsigned backend_disable_mask_per_asic;
  1761. unsigned backend_map;
  1762. unsigned num_texture_channel_caches;
  1763. unsigned mem_max_burst_length_bytes;
  1764. unsigned mem_row_size_in_kb;
  1765. unsigned shader_engine_tile_size;
  1766. unsigned num_gpus;
  1767. unsigned multi_gpu_tile_size;
  1768. unsigned tile_config;
  1769. uint32_t tile_mode_array[32];
  1770. };
  1771. struct cik_asic {
  1772. unsigned max_shader_engines;
  1773. unsigned max_tile_pipes;
  1774. unsigned max_cu_per_sh;
  1775. unsigned max_sh_per_se;
  1776. unsigned max_backends_per_se;
  1777. unsigned max_texture_channel_caches;
  1778. unsigned max_gprs;
  1779. unsigned max_gs_threads;
  1780. unsigned max_hw_contexts;
  1781. unsigned sc_prim_fifo_size_frontend;
  1782. unsigned sc_prim_fifo_size_backend;
  1783. unsigned sc_hiz_tile_fifo_size;
  1784. unsigned sc_earlyz_tile_fifo_size;
  1785. unsigned num_tile_pipes;
  1786. unsigned num_backends_per_se;
  1787. unsigned backend_disable_mask_per_asic;
  1788. unsigned backend_map;
  1789. unsigned num_texture_channel_caches;
  1790. unsigned mem_max_burst_length_bytes;
  1791. unsigned mem_row_size_in_kb;
  1792. unsigned shader_engine_tile_size;
  1793. unsigned num_gpus;
  1794. unsigned multi_gpu_tile_size;
  1795. unsigned tile_config;
  1796. uint32_t tile_mode_array[32];
  1797. uint32_t macrotile_mode_array[16];
  1798. };
  1799. union radeon_asic_config {
  1800. struct r300_asic r300;
  1801. struct r100_asic r100;
  1802. struct r600_asic r600;
  1803. struct rv770_asic rv770;
  1804. struct evergreen_asic evergreen;
  1805. struct cayman_asic cayman;
  1806. struct si_asic si;
  1807. struct cik_asic cik;
  1808. };
  1809. /*
  1810. * asic initizalization from radeon_asic.c
  1811. */
  1812. void radeon_agp_disable(struct radeon_device *rdev);
  1813. int radeon_asic_init(struct radeon_device *rdev);
  1814. /*
  1815. * IOCTL.
  1816. */
  1817. int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
  1818. struct drm_file *filp);
  1819. int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
  1820. struct drm_file *filp);
  1821. int radeon_gem_pin_ioctl(struct drm_device *dev, void *data,
  1822. struct drm_file *file_priv);
  1823. int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data,
  1824. struct drm_file *file_priv);
  1825. int radeon_gem_pwrite_ioctl(struct drm_device *dev, void *data,
  1826. struct drm_file *file_priv);
  1827. int radeon_gem_pread_ioctl(struct drm_device *dev, void *data,
  1828. struct drm_file *file_priv);
  1829. int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  1830. struct drm_file *filp);
  1831. int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
  1832. struct drm_file *filp);
  1833. int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
  1834. struct drm_file *filp);
  1835. int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
  1836. struct drm_file *filp);
  1837. int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
  1838. struct drm_file *filp);
  1839. int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
  1840. int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
  1841. struct drm_file *filp);
  1842. int radeon_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
  1843. struct drm_file *filp);
  1844. /* VRAM scratch page for HDP bug, default vram page */
  1845. struct r600_vram_scratch {
  1846. struct radeon_bo *robj;
  1847. volatile uint32_t *ptr;
  1848. u64 gpu_addr;
  1849. };
  1850. /*
  1851. * ACPI
  1852. */
  1853. struct radeon_atif_notification_cfg {
  1854. bool enabled;
  1855. int command_code;
  1856. };
  1857. struct radeon_atif_notifications {
  1858. bool display_switch;
  1859. bool expansion_mode_change;
  1860. bool thermal_state;
  1861. bool forced_power_state;
  1862. bool system_power_state;
  1863. bool display_conf_change;
  1864. bool px_gfx_switch;
  1865. bool brightness_change;
  1866. bool dgpu_display_event;
  1867. };
  1868. struct radeon_atif_functions {
  1869. bool system_params;
  1870. bool sbios_requests;
  1871. bool select_active_disp;
  1872. bool lid_state;
  1873. bool get_tv_standard;
  1874. bool set_tv_standard;
  1875. bool get_panel_expansion_mode;
  1876. bool set_panel_expansion_mode;
  1877. bool temperature_change;
  1878. bool graphics_device_types;
  1879. };
  1880. struct radeon_atif {
  1881. struct radeon_atif_notifications notifications;
  1882. struct radeon_atif_functions functions;
  1883. struct radeon_atif_notification_cfg notification_cfg;
  1884. struct radeon_encoder *encoder_for_bl;
  1885. };
  1886. struct radeon_atcs_functions {
  1887. bool get_ext_state;
  1888. bool pcie_perf_req;
  1889. bool pcie_dev_rdy;
  1890. bool pcie_bus_width;
  1891. };
  1892. struct radeon_atcs {
  1893. struct radeon_atcs_functions functions;
  1894. };
  1895. /*
  1896. * Core structure, functions and helpers.
  1897. */
  1898. typedef uint32_t (*radeon_rreg_t)(struct radeon_device*, uint32_t);
  1899. typedef void (*radeon_wreg_t)(struct radeon_device*, uint32_t, uint32_t);
  1900. struct radeon_device {
  1901. struct device *dev;
  1902. struct drm_device *ddev;
  1903. struct pci_dev *pdev;
  1904. struct rw_semaphore exclusive_lock;
  1905. /* ASIC */
  1906. union radeon_asic_config config;
  1907. enum radeon_family family;
  1908. unsigned long flags;
  1909. int usec_timeout;
  1910. enum radeon_pll_errata pll_errata;
  1911. int num_gb_pipes;
  1912. int num_z_pipes;
  1913. int disp_priority;
  1914. /* BIOS */
  1915. uint8_t *bios;
  1916. bool is_atom_bios;
  1917. uint16_t bios_header_start;
  1918. struct radeon_bo *stollen_vga_memory;
  1919. /* Register mmio */
  1920. resource_size_t rmmio_base;
  1921. resource_size_t rmmio_size;
  1922. /* protects concurrent MM_INDEX/DATA based register access */
  1923. spinlock_t mmio_idx_lock;
  1924. /* protects concurrent SMC based register access */
  1925. spinlock_t smc_idx_lock;
  1926. /* protects concurrent PLL register access */
  1927. spinlock_t pll_idx_lock;
  1928. /* protects concurrent MC register access */
  1929. spinlock_t mc_idx_lock;
  1930. /* protects concurrent PCIE register access */
  1931. spinlock_t pcie_idx_lock;
  1932. /* protects concurrent PCIE_PORT register access */
  1933. spinlock_t pciep_idx_lock;
  1934. /* protects concurrent PIF register access */
  1935. spinlock_t pif_idx_lock;
  1936. /* protects concurrent CG register access */
  1937. spinlock_t cg_idx_lock;
  1938. /* protects concurrent UVD register access */
  1939. spinlock_t uvd_idx_lock;
  1940. /* protects concurrent RCU register access */
  1941. spinlock_t rcu_idx_lock;
  1942. /* protects concurrent DIDT register access */
  1943. spinlock_t didt_idx_lock;
  1944. /* protects concurrent ENDPOINT (audio) register access */
  1945. spinlock_t end_idx_lock;
  1946. void __iomem *rmmio;
  1947. radeon_rreg_t mc_rreg;
  1948. radeon_wreg_t mc_wreg;
  1949. radeon_rreg_t pll_rreg;
  1950. radeon_wreg_t pll_wreg;
  1951. uint32_t pcie_reg_mask;
  1952. radeon_rreg_t pciep_rreg;
  1953. radeon_wreg_t pciep_wreg;
  1954. /* io port */
  1955. void __iomem *rio_mem;
  1956. resource_size_t rio_mem_size;
  1957. struct radeon_clock clock;
  1958. struct radeon_mc mc;
  1959. struct radeon_gart gart;
  1960. struct radeon_mode_info mode_info;
  1961. struct radeon_scratch scratch;
  1962. struct radeon_doorbell doorbell;
  1963. struct radeon_mman mman;
  1964. struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS];
  1965. wait_queue_head_t fence_queue;
  1966. struct mutex ring_lock;
  1967. struct radeon_ring ring[RADEON_NUM_RINGS];
  1968. bool ib_pool_ready;
  1969. struct radeon_sa_manager ring_tmp_bo;
  1970. struct radeon_irq irq;
  1971. struct radeon_asic *asic;
  1972. struct radeon_gem gem;
  1973. struct radeon_pm pm;
  1974. struct radeon_uvd uvd;
  1975. uint32_t bios_scratch[RADEON_BIOS_NUM_SCRATCH];
  1976. struct radeon_wb wb;
  1977. struct radeon_dummy_page dummy_page;
  1978. bool shutdown;
  1979. bool suspend;
  1980. bool need_dma32;
  1981. bool accel_working;
  1982. bool fastfb_working; /* IGP feature*/
  1983. bool needs_reset;
  1984. struct radeon_surface_reg surface_regs[RADEON_GEM_MAX_SURFACES];
  1985. const struct firmware *me_fw; /* all family ME firmware */
  1986. const struct firmware *pfp_fw; /* r6/700 PFP firmware */
  1987. const struct firmware *rlc_fw; /* r6/700 RLC firmware */
  1988. const struct firmware *mc_fw; /* NI MC firmware */
  1989. const struct firmware *ce_fw; /* SI CE firmware */
  1990. const struct firmware *mec_fw; /* CIK MEC firmware */
  1991. const struct firmware *sdma_fw; /* CIK SDMA firmware */
  1992. const struct firmware *smc_fw; /* SMC firmware */
  1993. const struct firmware *uvd_fw; /* UVD firmware */
  1994. struct r600_vram_scratch vram_scratch;
  1995. int msi_enabled; /* msi enabled */
  1996. struct r600_ih ih; /* r6/700 interrupt ring */
  1997. struct radeon_rlc rlc;
  1998. struct radeon_mec mec;
  1999. struct work_struct hotplug_work;
  2000. struct work_struct audio_work;
  2001. struct work_struct reset_work;
  2002. int num_crtc; /* number of crtcs */
  2003. struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
  2004. bool has_uvd;
  2005. struct r600_audio audio; /* audio stuff */
  2006. struct notifier_block acpi_nb;
  2007. /* only one userspace can use Hyperz features or CMASK at a time */
  2008. struct drm_file *hyperz_filp;
  2009. struct drm_file *cmask_filp;
  2010. /* i2c buses */
  2011. struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
  2012. /* debugfs */
  2013. struct radeon_debugfs debugfs[RADEON_DEBUGFS_MAX_COMPONENTS];
  2014. unsigned debugfs_count;
  2015. /* virtual memory */
  2016. struct radeon_vm_manager vm_manager;
  2017. struct mutex gpu_clock_mutex;
  2018. /* ACPI interface */
  2019. struct radeon_atif atif;
  2020. struct radeon_atcs atcs;
  2021. /* srbm instance registers */
  2022. struct mutex srbm_mutex;
  2023. /* clock, powergating flags */
  2024. u32 cg_flags;
  2025. u32 pg_flags;
  2026. struct dev_pm_domain vga_pm_domain;
  2027. bool have_disp_power_ref;
  2028. };
  2029. int radeon_device_init(struct radeon_device *rdev,
  2030. struct drm_device *ddev,
  2031. struct pci_dev *pdev,
  2032. uint32_t flags);
  2033. void radeon_device_fini(struct radeon_device *rdev);
  2034. int radeon_gpu_wait_for_idle(struct radeon_device *rdev);
  2035. uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg,
  2036. bool always_indirect);
  2037. void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v,
  2038. bool always_indirect);
  2039. u32 r100_io_rreg(struct radeon_device *rdev, u32 reg);
  2040. void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v);
  2041. u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index);
  2042. void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v);
  2043. /*
  2044. * Cast helper
  2045. */
  2046. #define to_radeon_fence(p) ((struct radeon_fence *)(p))
  2047. /*
  2048. * Registers read & write functions.
  2049. */
  2050. #define RREG8(reg) readb((rdev->rmmio) + (reg))
  2051. #define WREG8(reg, v) writeb(v, (rdev->rmmio) + (reg))
  2052. #define RREG16(reg) readw((rdev->rmmio) + (reg))
  2053. #define WREG16(reg, v) writew(v, (rdev->rmmio) + (reg))
  2054. #define RREG32(reg) r100_mm_rreg(rdev, (reg), false)
  2055. #define RREG32_IDX(reg) r100_mm_rreg(rdev, (reg), true)
  2056. #define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg), false))
  2057. #define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v), false)
  2058. #define WREG32_IDX(reg, v) r100_mm_wreg(rdev, (reg), (v), true)
  2059. #define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
  2060. #define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
  2061. #define RREG32_PLL(reg) rdev->pll_rreg(rdev, (reg))
  2062. #define WREG32_PLL(reg, v) rdev->pll_wreg(rdev, (reg), (v))
  2063. #define RREG32_MC(reg) rdev->mc_rreg(rdev, (reg))
  2064. #define WREG32_MC(reg, v) rdev->mc_wreg(rdev, (reg), (v))
  2065. #define RREG32_PCIE(reg) rv370_pcie_rreg(rdev, (reg))
  2066. #define WREG32_PCIE(reg, v) rv370_pcie_wreg(rdev, (reg), (v))
  2067. #define RREG32_PCIE_PORT(reg) rdev->pciep_rreg(rdev, (reg))
  2068. #define WREG32_PCIE_PORT(reg, v) rdev->pciep_wreg(rdev, (reg), (v))
  2069. #define RREG32_SMC(reg) tn_smc_rreg(rdev, (reg))
  2070. #define WREG32_SMC(reg, v) tn_smc_wreg(rdev, (reg), (v))
  2071. #define RREG32_RCU(reg) r600_rcu_rreg(rdev, (reg))
  2072. #define WREG32_RCU(reg, v) r600_rcu_wreg(rdev, (reg), (v))
  2073. #define RREG32_CG(reg) eg_cg_rreg(rdev, (reg))
  2074. #define WREG32_CG(reg, v) eg_cg_wreg(rdev, (reg), (v))
  2075. #define RREG32_PIF_PHY0(reg) eg_pif_phy0_rreg(rdev, (reg))
  2076. #define WREG32_PIF_PHY0(reg, v) eg_pif_phy0_wreg(rdev, (reg), (v))
  2077. #define RREG32_PIF_PHY1(reg) eg_pif_phy1_rreg(rdev, (reg))
  2078. #define WREG32_PIF_PHY1(reg, v) eg_pif_phy1_wreg(rdev, (reg), (v))
  2079. #define RREG32_UVD_CTX(reg) r600_uvd_ctx_rreg(rdev, (reg))
  2080. #define WREG32_UVD_CTX(reg, v) r600_uvd_ctx_wreg(rdev, (reg), (v))
  2081. #define RREG32_DIDT(reg) cik_didt_rreg(rdev, (reg))
  2082. #define WREG32_DIDT(reg, v) cik_didt_wreg(rdev, (reg), (v))
  2083. #define WREG32_P(reg, val, mask) \
  2084. do { \
  2085. uint32_t tmp_ = RREG32(reg); \
  2086. tmp_ &= (mask); \
  2087. tmp_ |= ((val) & ~(mask)); \
  2088. WREG32(reg, tmp_); \
  2089. } while (0)
  2090. #define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
  2091. #define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
  2092. #define WREG32_PLL_P(reg, val, mask) \
  2093. do { \
  2094. uint32_t tmp_ = RREG32_PLL(reg); \
  2095. tmp_ &= (mask); \
  2096. tmp_ |= ((val) & ~(mask)); \
  2097. WREG32_PLL(reg, tmp_); \
  2098. } while (0)
  2099. #define DREG32_SYS(sqf, rdev, reg) seq_printf((sqf), #reg " : 0x%08X\n", r100_mm_rreg((rdev), (reg), false))
  2100. #define RREG32_IO(reg) r100_io_rreg(rdev, (reg))
  2101. #define WREG32_IO(reg, v) r100_io_wreg(rdev, (reg), (v))
  2102. #define RDOORBELL32(index) cik_mm_rdoorbell(rdev, (index))
  2103. #define WDOORBELL32(index, v) cik_mm_wdoorbell(rdev, (index), (v))
  2104. /*
  2105. * Indirect registers accessor
  2106. */
  2107. static inline uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg)
  2108. {
  2109. unsigned long flags;
  2110. uint32_t r;
  2111. spin_lock_irqsave(&rdev->pcie_idx_lock, flags);
  2112. WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask));
  2113. r = RREG32(RADEON_PCIE_DATA);
  2114. spin_unlock_irqrestore(&rdev->pcie_idx_lock, flags);
  2115. return r;
  2116. }
  2117. static inline void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
  2118. {
  2119. unsigned long flags;
  2120. spin_lock_irqsave(&rdev->pcie_idx_lock, flags);
  2121. WREG32(RADEON_PCIE_INDEX, ((reg) & rdev->pcie_reg_mask));
  2122. WREG32(RADEON_PCIE_DATA, (v));
  2123. spin_unlock_irqrestore(&rdev->pcie_idx_lock, flags);
  2124. }
  2125. static inline u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  2126. {
  2127. unsigned long flags;
  2128. u32 r;
  2129. spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  2130. WREG32(TN_SMC_IND_INDEX_0, (reg));
  2131. r = RREG32(TN_SMC_IND_DATA_0);
  2132. spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  2133. return r;
  2134. }
  2135. static inline void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2136. {
  2137. unsigned long flags;
  2138. spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  2139. WREG32(TN_SMC_IND_INDEX_0, (reg));
  2140. WREG32(TN_SMC_IND_DATA_0, (v));
  2141. spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  2142. }
  2143. static inline u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg)
  2144. {
  2145. unsigned long flags;
  2146. u32 r;
  2147. spin_lock_irqsave(&rdev->rcu_idx_lock, flags);
  2148. WREG32(R600_RCU_INDEX, ((reg) & 0x1fff));
  2149. r = RREG32(R600_RCU_DATA);
  2150. spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags);
  2151. return r;
  2152. }
  2153. static inline void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2154. {
  2155. unsigned long flags;
  2156. spin_lock_irqsave(&rdev->rcu_idx_lock, flags);
  2157. WREG32(R600_RCU_INDEX, ((reg) & 0x1fff));
  2158. WREG32(R600_RCU_DATA, (v));
  2159. spin_unlock_irqrestore(&rdev->rcu_idx_lock, flags);
  2160. }
  2161. static inline u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  2162. {
  2163. unsigned long flags;
  2164. u32 r;
  2165. spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  2166. WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  2167. r = RREG32(EVERGREEN_CG_IND_DATA);
  2168. spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  2169. return r;
  2170. }
  2171. static inline void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2172. {
  2173. unsigned long flags;
  2174. spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  2175. WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  2176. WREG32(EVERGREEN_CG_IND_DATA, (v));
  2177. spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  2178. }
  2179. static inline u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  2180. {
  2181. unsigned long flags;
  2182. u32 r;
  2183. spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  2184. WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  2185. r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  2186. spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  2187. return r;
  2188. }
  2189. static inline void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2190. {
  2191. unsigned long flags;
  2192. spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  2193. WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  2194. WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  2195. spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  2196. }
  2197. static inline u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  2198. {
  2199. unsigned long flags;
  2200. u32 r;
  2201. spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  2202. WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
  2203. r = RREG32(EVERGREEN_PIF_PHY1_DATA);
  2204. spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  2205. return r;
  2206. }
  2207. static inline void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2208. {
  2209. unsigned long flags;
  2210. spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  2211. WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
  2212. WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
  2213. spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  2214. }
  2215. static inline u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg)
  2216. {
  2217. unsigned long flags;
  2218. u32 r;
  2219. spin_lock_irqsave(&rdev->uvd_idx_lock, flags);
  2220. WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff));
  2221. r = RREG32(R600_UVD_CTX_DATA);
  2222. spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags);
  2223. return r;
  2224. }
  2225. static inline void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2226. {
  2227. unsigned long flags;
  2228. spin_lock_irqsave(&rdev->uvd_idx_lock, flags);
  2229. WREG32(R600_UVD_CTX_INDEX, ((reg) & 0x1ff));
  2230. WREG32(R600_UVD_CTX_DATA, (v));
  2231. spin_unlock_irqrestore(&rdev->uvd_idx_lock, flags);
  2232. }
  2233. static inline u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
  2234. {
  2235. unsigned long flags;
  2236. u32 r;
  2237. spin_lock_irqsave(&rdev->didt_idx_lock, flags);
  2238. WREG32(CIK_DIDT_IND_INDEX, (reg));
  2239. r = RREG32(CIK_DIDT_IND_DATA);
  2240. spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
  2241. return r;
  2242. }
  2243. static inline void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  2244. {
  2245. unsigned long flags;
  2246. spin_lock_irqsave(&rdev->didt_idx_lock, flags);
  2247. WREG32(CIK_DIDT_IND_INDEX, (reg));
  2248. WREG32(CIK_DIDT_IND_DATA, (v));
  2249. spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
  2250. }
  2251. void r100_pll_errata_after_index(struct radeon_device *rdev);
  2252. /*
  2253. * ASICs helpers.
  2254. */
  2255. #define ASIC_IS_RN50(rdev) ((rdev->pdev->device == 0x515e) || \
  2256. (rdev->pdev->device == 0x5969))
  2257. #define ASIC_IS_RV100(rdev) ((rdev->family == CHIP_RV100) || \
  2258. (rdev->family == CHIP_RV200) || \
  2259. (rdev->family == CHIP_RS100) || \
  2260. (rdev->family == CHIP_RS200) || \
  2261. (rdev->family == CHIP_RV250) || \
  2262. (rdev->family == CHIP_RV280) || \
  2263. (rdev->family == CHIP_RS300))
  2264. #define ASIC_IS_R300(rdev) ((rdev->family == CHIP_R300) || \
  2265. (rdev->family == CHIP_RV350) || \
  2266. (rdev->family == CHIP_R350) || \
  2267. (rdev->family == CHIP_RV380) || \
  2268. (rdev->family == CHIP_R420) || \
  2269. (rdev->family == CHIP_R423) || \
  2270. (rdev->family == CHIP_RV410) || \
  2271. (rdev->family == CHIP_RS400) || \
  2272. (rdev->family == CHIP_RS480))
  2273. #define ASIC_IS_X2(rdev) ((rdev->ddev->pdev->device == 0x9441) || \
  2274. (rdev->ddev->pdev->device == 0x9443) || \
  2275. (rdev->ddev->pdev->device == 0x944B) || \
  2276. (rdev->ddev->pdev->device == 0x9506) || \
  2277. (rdev->ddev->pdev->device == 0x9509) || \
  2278. (rdev->ddev->pdev->device == 0x950F) || \
  2279. (rdev->ddev->pdev->device == 0x689C) || \
  2280. (rdev->ddev->pdev->device == 0x689D))
  2281. #define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600))
  2282. #define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600) || \
  2283. (rdev->family == CHIP_RS690) || \
  2284. (rdev->family == CHIP_RS740) || \
  2285. (rdev->family >= CHIP_R600))
  2286. #define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620))
  2287. #define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730))
  2288. #define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR))
  2289. #define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM) && \
  2290. (rdev->flags & RADEON_IS_IGP))
  2291. #define ASIC_IS_DCE5(rdev) ((rdev->family >= CHIP_BARTS))
  2292. #define ASIC_IS_DCE6(rdev) ((rdev->family >= CHIP_ARUBA))
  2293. #define ASIC_IS_DCE61(rdev) ((rdev->family >= CHIP_ARUBA) && \
  2294. (rdev->flags & RADEON_IS_IGP))
  2295. #define ASIC_IS_DCE64(rdev) ((rdev->family == CHIP_OLAND))
  2296. #define ASIC_IS_NODCE(rdev) ((rdev->family == CHIP_HAINAN))
  2297. #define ASIC_IS_DCE8(rdev) ((rdev->family >= CHIP_BONAIRE))
  2298. #define ASIC_IS_LOMBOK(rdev) ((rdev->ddev->pdev->device == 0x6849) || \
  2299. (rdev->ddev->pdev->device == 0x6850) || \
  2300. (rdev->ddev->pdev->device == 0x6858) || \
  2301. (rdev->ddev->pdev->device == 0x6859) || \
  2302. (rdev->ddev->pdev->device == 0x6840) || \
  2303. (rdev->ddev->pdev->device == 0x6841) || \
  2304. (rdev->ddev->pdev->device == 0x6842) || \
  2305. (rdev->ddev->pdev->device == 0x6843))
  2306. /*
  2307. * BIOS helpers.
  2308. */
  2309. #define RBIOS8(i) (rdev->bios[i])
  2310. #define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
  2311. #define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
  2312. int radeon_combios_init(struct radeon_device *rdev);
  2313. void radeon_combios_fini(struct radeon_device *rdev);
  2314. int radeon_atombios_init(struct radeon_device *rdev);
  2315. void radeon_atombios_fini(struct radeon_device *rdev);
  2316. /*
  2317. * RING helpers.
  2318. */
  2319. #if DRM_DEBUG_CODE == 0
  2320. static inline void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
  2321. {
  2322. ring->ring[ring->wptr++] = v;
  2323. ring->wptr &= ring->ptr_mask;
  2324. ring->count_dw--;
  2325. ring->ring_free_dw--;
  2326. }
  2327. #else
  2328. /* With debugging this is just too big to inline */
  2329. void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
  2330. #endif
  2331. /*
  2332. * ASICs macro.
  2333. */
  2334. #define radeon_init(rdev) (rdev)->asic->init((rdev))
  2335. #define radeon_fini(rdev) (rdev)->asic->fini((rdev))
  2336. #define radeon_resume(rdev) (rdev)->asic->resume((rdev))
  2337. #define radeon_suspend(rdev) (rdev)->asic->suspend((rdev))
  2338. #define radeon_cs_parse(rdev, r, p) (rdev)->asic->ring[(r)]->cs_parse((p))
  2339. #define radeon_vga_set_state(rdev, state) (rdev)->asic->vga_set_state((rdev), (state))
  2340. #define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev))
  2341. #define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart.tlb_flush((rdev))
  2342. #define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart.set_page((rdev), (i), (p))
  2343. #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev))
  2344. #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev))
  2345. #define radeon_asic_vm_set_page(rdev, ib, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_page((rdev), (ib), (pe), (addr), (count), (incr), (flags)))
  2346. #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)]->ring_start((rdev), (cp))
  2347. #define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)]->ring_test((rdev), (cp))
  2348. #define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)]->ib_test((rdev), (cp))
  2349. #define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_execute((rdev), (ib))
  2350. #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_parse((rdev), (ib))
  2351. #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)]->is_lockup((rdev), (cp))
  2352. #define radeon_ring_vm_flush(rdev, r, vm) (rdev)->asic->ring[(r)]->vm_flush((rdev), (r), (vm))
  2353. #define radeon_ring_get_rptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_rptr((rdev), (r))
  2354. #define radeon_ring_get_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_wptr((rdev), (r))
  2355. #define radeon_ring_set_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->set_wptr((rdev), (r))
  2356. #define radeon_irq_set(rdev) (rdev)->asic->irq.set((rdev))
  2357. #define radeon_irq_process(rdev) (rdev)->asic->irq.process((rdev))
  2358. #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->display.get_vblank_counter((rdev), (crtc))
  2359. #define radeon_set_backlight_level(rdev, e, l) (rdev)->asic->display.set_backlight_level((e), (l))
  2360. #define radeon_get_backlight_level(rdev, e) (rdev)->asic->display.get_backlight_level((e))
  2361. #define radeon_hdmi_enable(rdev, e, b) (rdev)->asic->display.hdmi_enable((e), (b))
  2362. #define radeon_hdmi_setmode(rdev, e, m) (rdev)->asic->display.hdmi_setmode((e), (m))
  2363. #define radeon_fence_ring_emit(rdev, r, fence) (rdev)->asic->ring[(r)]->emit_fence((rdev), (fence))
  2364. #define radeon_semaphore_ring_emit(rdev, r, cp, semaphore, emit_wait) (rdev)->asic->ring[(r)]->emit_semaphore((rdev), (cp), (semaphore), (emit_wait))
  2365. #define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy.blit((rdev), (s), (d), (np), (f))
  2366. #define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy.dma((rdev), (s), (d), (np), (f))
  2367. #define radeon_copy(rdev, s, d, np, f) (rdev)->asic->copy.copy((rdev), (s), (d), (np), (f))
  2368. #define radeon_copy_blit_ring_index(rdev) (rdev)->asic->copy.blit_ring_index
  2369. #define radeon_copy_dma_ring_index(rdev) (rdev)->asic->copy.dma_ring_index
  2370. #define radeon_copy_ring_index(rdev) (rdev)->asic->copy.copy_ring_index
  2371. #define radeon_get_engine_clock(rdev) (rdev)->asic->pm.get_engine_clock((rdev))
  2372. #define radeon_set_engine_clock(rdev, e) (rdev)->asic->pm.set_engine_clock((rdev), (e))
  2373. #define radeon_get_memory_clock(rdev) (rdev)->asic->pm.get_memory_clock((rdev))
  2374. #define radeon_set_memory_clock(rdev, e) (rdev)->asic->pm.set_memory_clock((rdev), (e))
  2375. #define radeon_get_pcie_lanes(rdev) (rdev)->asic->pm.get_pcie_lanes((rdev))
  2376. #define radeon_set_pcie_lanes(rdev, l) (rdev)->asic->pm.set_pcie_lanes((rdev), (l))
  2377. #define radeon_set_clock_gating(rdev, e) (rdev)->asic->pm.set_clock_gating((rdev), (e))
  2378. #define radeon_set_uvd_clocks(rdev, v, d) (rdev)->asic->pm.set_uvd_clocks((rdev), (v), (d))
  2379. #define radeon_get_temperature(rdev) (rdev)->asic->pm.get_temperature((rdev))
  2380. #define radeon_set_surface_reg(rdev, r, f, p, o, s) ((rdev)->asic->surface.set_reg((rdev), (r), (f), (p), (o), (s)))
  2381. #define radeon_clear_surface_reg(rdev, r) ((rdev)->asic->surface.clear_reg((rdev), (r)))
  2382. #define radeon_bandwidth_update(rdev) (rdev)->asic->display.bandwidth_update((rdev))
  2383. #define radeon_hpd_init(rdev) (rdev)->asic->hpd.init((rdev))
  2384. #define radeon_hpd_fini(rdev) (rdev)->asic->hpd.fini((rdev))
  2385. #define radeon_hpd_sense(rdev, h) (rdev)->asic->hpd.sense((rdev), (h))
  2386. #define radeon_hpd_set_polarity(rdev, h) (rdev)->asic->hpd.set_polarity((rdev), (h))
  2387. #define radeon_gui_idle(rdev) (rdev)->asic->gui_idle((rdev))
  2388. #define radeon_pm_misc(rdev) (rdev)->asic->pm.misc((rdev))
  2389. #define radeon_pm_prepare(rdev) (rdev)->asic->pm.prepare((rdev))
  2390. #define radeon_pm_finish(rdev) (rdev)->asic->pm.finish((rdev))
  2391. #define radeon_pm_init_profile(rdev) (rdev)->asic->pm.init_profile((rdev))
  2392. #define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm.get_dynpm_state((rdev))
  2393. #define radeon_pre_page_flip(rdev, crtc) (rdev)->asic->pflip.pre_page_flip((rdev), (crtc))
  2394. #define radeon_page_flip(rdev, crtc, base) (rdev)->asic->pflip.page_flip((rdev), (crtc), (base))
  2395. #define radeon_post_page_flip(rdev, crtc) (rdev)->asic->pflip.post_page_flip((rdev), (crtc))
  2396. #define radeon_wait_for_vblank(rdev, crtc) (rdev)->asic->display.wait_for_vblank((rdev), (crtc))
  2397. #define radeon_mc_wait_for_idle(rdev) (rdev)->asic->mc_wait_for_idle((rdev))
  2398. #define radeon_get_xclk(rdev) (rdev)->asic->get_xclk((rdev))
  2399. #define radeon_get_gpu_clock_counter(rdev) (rdev)->asic->get_gpu_clock_counter((rdev))
  2400. #define radeon_dpm_init(rdev) rdev->asic->dpm.init((rdev))
  2401. #define radeon_dpm_setup_asic(rdev) rdev->asic->dpm.setup_asic((rdev))
  2402. #define radeon_dpm_enable(rdev) rdev->asic->dpm.enable((rdev))
  2403. #define radeon_dpm_disable(rdev) rdev->asic->dpm.disable((rdev))
  2404. #define radeon_dpm_pre_set_power_state(rdev) rdev->asic->dpm.pre_set_power_state((rdev))
  2405. #define radeon_dpm_set_power_state(rdev) rdev->asic->dpm.set_power_state((rdev))
  2406. #define radeon_dpm_post_set_power_state(rdev) rdev->asic->dpm.post_set_power_state((rdev))
  2407. #define radeon_dpm_display_configuration_changed(rdev) rdev->asic->dpm.display_configuration_changed((rdev))
  2408. #define radeon_dpm_fini(rdev) rdev->asic->dpm.fini((rdev))
  2409. #define radeon_dpm_get_sclk(rdev, l) rdev->asic->dpm.get_sclk((rdev), (l))
  2410. #define radeon_dpm_get_mclk(rdev, l) rdev->asic->dpm.get_mclk((rdev), (l))
  2411. #define radeon_dpm_print_power_state(rdev, ps) rdev->asic->dpm.print_power_state((rdev), (ps))
  2412. #define radeon_dpm_debugfs_print_current_performance_level(rdev, m) rdev->asic->dpm.debugfs_print_current_performance_level((rdev), (m))
  2413. #define radeon_dpm_force_performance_level(rdev, l) rdev->asic->dpm.force_performance_level((rdev), (l))
  2414. #define radeon_dpm_vblank_too_short(rdev) rdev->asic->dpm.vblank_too_short((rdev))
  2415. #define radeon_dpm_powergate_uvd(rdev, g) rdev->asic->dpm.powergate_uvd((rdev), (g))
  2416. #define radeon_dpm_enable_bapm(rdev, e) rdev->asic->dpm.enable_bapm((rdev), (e))
  2417. /* Common functions */
  2418. /* AGP */
  2419. extern int radeon_gpu_reset(struct radeon_device *rdev);
  2420. extern void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung);
  2421. extern void radeon_agp_disable(struct radeon_device *rdev);
  2422. extern int radeon_modeset_init(struct radeon_device *rdev);
  2423. extern void radeon_modeset_fini(struct radeon_device *rdev);
  2424. extern bool radeon_card_posted(struct radeon_device *rdev);
  2425. extern void radeon_update_bandwidth_info(struct radeon_device *rdev);
  2426. extern void radeon_update_display_priority(struct radeon_device *rdev);
  2427. extern bool radeon_boot_test_post_card(struct radeon_device *rdev);
  2428. extern void radeon_scratch_init(struct radeon_device *rdev);
  2429. extern void radeon_wb_fini(struct radeon_device *rdev);
  2430. extern int radeon_wb_init(struct radeon_device *rdev);
  2431. extern void radeon_wb_disable(struct radeon_device *rdev);
  2432. extern void radeon_surface_init(struct radeon_device *rdev);
  2433. extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data);
  2434. extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable);
  2435. extern void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable);
  2436. extern void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain);
  2437. extern bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo);
  2438. extern void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base);
  2439. extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  2440. extern int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
  2441. extern int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon);
  2442. extern void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size);
  2443. extern void radeon_program_register_sequence(struct radeon_device *rdev,
  2444. const u32 *registers,
  2445. const u32 array_size);
  2446. /*
  2447. * vm
  2448. */
  2449. int radeon_vm_manager_init(struct radeon_device *rdev);
  2450. void radeon_vm_manager_fini(struct radeon_device *rdev);
  2451. void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm);
  2452. void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm);
  2453. int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm);
  2454. void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm);
  2455. struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
  2456. struct radeon_vm *vm, int ring);
  2457. void radeon_vm_fence(struct radeon_device *rdev,
  2458. struct radeon_vm *vm,
  2459. struct radeon_fence *fence);
  2460. uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr);
  2461. int radeon_vm_bo_update_pte(struct radeon_device *rdev,
  2462. struct radeon_vm *vm,
  2463. struct radeon_bo *bo,
  2464. struct ttm_mem_reg *mem);
  2465. void radeon_vm_bo_invalidate(struct radeon_device *rdev,
  2466. struct radeon_bo *bo);
  2467. struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
  2468. struct radeon_bo *bo);
  2469. struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
  2470. struct radeon_vm *vm,
  2471. struct radeon_bo *bo);
  2472. int radeon_vm_bo_set_addr(struct radeon_device *rdev,
  2473. struct radeon_bo_va *bo_va,
  2474. uint64_t offset,
  2475. uint32_t flags);
  2476. int radeon_vm_bo_rmv(struct radeon_device *rdev,
  2477. struct radeon_bo_va *bo_va);
  2478. /* audio */
  2479. void r600_audio_update_hdmi(struct work_struct *work);
  2480. struct r600_audio_pin *r600_audio_get_pin(struct radeon_device *rdev);
  2481. struct r600_audio_pin *dce6_audio_get_pin(struct radeon_device *rdev);
  2482. /*
  2483. * R600 vram scratch functions
  2484. */
  2485. int r600_vram_scratch_init(struct radeon_device *rdev);
  2486. void r600_vram_scratch_fini(struct radeon_device *rdev);
  2487. /*
  2488. * r600 cs checking helper
  2489. */
  2490. unsigned r600_mip_minify(unsigned size, unsigned level);
  2491. bool r600_fmt_is_valid_color(u32 format);
  2492. bool r600_fmt_is_valid_texture(u32 format, enum radeon_family family);
  2493. int r600_fmt_get_blocksize(u32 format);
  2494. int r600_fmt_get_nblocksx(u32 format, u32 w);
  2495. int r600_fmt_get_nblocksy(u32 format, u32 h);
  2496. /*
  2497. * r600 functions used by radeon_encoder.c
  2498. */
  2499. struct radeon_hdmi_acr {
  2500. u32 clock;
  2501. int n_32khz;
  2502. int cts_32khz;
  2503. int n_44_1khz;
  2504. int cts_44_1khz;
  2505. int n_48khz;
  2506. int cts_48khz;
  2507. };
  2508. extern struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock);
  2509. extern u32 r6xx_remap_render_backend(struct radeon_device *rdev,
  2510. u32 tiling_pipe_num,
  2511. u32 max_rb_num,
  2512. u32 total_max_rb_num,
  2513. u32 enabled_rb_mask);
  2514. /*
  2515. * evergreen functions used by radeon_encoder.c
  2516. */
  2517. extern int ni_init_microcode(struct radeon_device *rdev);
  2518. extern int ni_mc_load_microcode(struct radeon_device *rdev);
  2519. /* radeon_acpi.c */
  2520. #if defined(CONFIG_ACPI)
  2521. extern int radeon_acpi_init(struct radeon_device *rdev);
  2522. extern void radeon_acpi_fini(struct radeon_device *rdev);
  2523. extern bool radeon_acpi_is_pcie_performance_request_supported(struct radeon_device *rdev);
  2524. extern int radeon_acpi_pcie_performance_request(struct radeon_device *rdev,
  2525. u8 perf_req, bool advertise);
  2526. extern int radeon_acpi_pcie_notify_device_ready(struct radeon_device *rdev);
  2527. #else
  2528. static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; }
  2529. static inline void radeon_acpi_fini(struct radeon_device *rdev) { }
  2530. #endif
  2531. int radeon_cs_packet_parse(struct radeon_cs_parser *p,
  2532. struct radeon_cs_packet *pkt,
  2533. unsigned idx);
  2534. bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p);
  2535. void radeon_cs_dump_packet(struct radeon_cs_parser *p,
  2536. struct radeon_cs_packet *pkt);
  2537. int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
  2538. struct radeon_cs_reloc **cs_reloc,
  2539. int nomm);
  2540. int r600_cs_common_vline_parse(struct radeon_cs_parser *p,
  2541. uint32_t *vline_start_end,
  2542. uint32_t *vline_status);
  2543. #include "radeon_object.h"
  2544. #endif