mmu.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309
  1. /*
  2. * linux/arch/arm/mm/mmu.c
  3. *
  4. * Copyright (C) 1995-2005 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/kernel.h>
  12. #include <linux/errno.h>
  13. #include <linux/init.h>
  14. #include <linux/mman.h>
  15. #include <linux/nodemask.h>
  16. #include <linux/memblock.h>
  17. #include <linux/fs.h>
  18. #include <linux/vmalloc.h>
  19. #include <linux/sizes.h>
  20. #include <asm/cp15.h>
  21. #include <asm/cputype.h>
  22. #include <asm/sections.h>
  23. #include <asm/cachetype.h>
  24. #include <asm/setup.h>
  25. #include <asm/smp_plat.h>
  26. #include <asm/tlb.h>
  27. #include <asm/highmem.h>
  28. #include <asm/system_info.h>
  29. #include <asm/traps.h>
  30. #include <asm/mach/arch.h>
  31. #include <asm/mach/map.h>
  32. #include <asm/mach/pci.h>
  33. #include "mm.h"
  34. #include "tcm.h"
  35. /*
  36. * empty_zero_page is a special page that is used for
  37. * zero-initialized data and COW.
  38. */
  39. struct page *empty_zero_page;
  40. EXPORT_SYMBOL(empty_zero_page);
  41. /*
  42. * The pmd table for the upper-most set of pages.
  43. */
  44. pmd_t *top_pmd;
  45. #define CPOLICY_UNCACHED 0
  46. #define CPOLICY_BUFFERED 1
  47. #define CPOLICY_WRITETHROUGH 2
  48. #define CPOLICY_WRITEBACK 3
  49. #define CPOLICY_WRITEALLOC 4
  50. static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
  51. static unsigned int ecc_mask __initdata = 0;
  52. pgprot_t pgprot_user;
  53. pgprot_t pgprot_kernel;
  54. pgprot_t pgprot_hyp_device;
  55. pgprot_t pgprot_s2;
  56. pgprot_t pgprot_s2_device;
  57. EXPORT_SYMBOL(pgprot_user);
  58. EXPORT_SYMBOL(pgprot_kernel);
  59. struct cachepolicy {
  60. const char policy[16];
  61. unsigned int cr_mask;
  62. pmdval_t pmd;
  63. pteval_t pte;
  64. pteval_t pte_s2;
  65. };
  66. #ifdef CONFIG_ARM_LPAE
  67. #define s2_policy(policy) policy
  68. #else
  69. #define s2_policy(policy) 0
  70. #endif
  71. static struct cachepolicy cache_policies[] __initdata = {
  72. {
  73. .policy = "uncached",
  74. .cr_mask = CR_W|CR_C,
  75. .pmd = PMD_SECT_UNCACHED,
  76. .pte = L_PTE_MT_UNCACHED,
  77. .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
  78. }, {
  79. .policy = "buffered",
  80. .cr_mask = CR_C,
  81. .pmd = PMD_SECT_BUFFERED,
  82. .pte = L_PTE_MT_BUFFERABLE,
  83. .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
  84. }, {
  85. .policy = "writethrough",
  86. .cr_mask = 0,
  87. .pmd = PMD_SECT_WT,
  88. .pte = L_PTE_MT_WRITETHROUGH,
  89. .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH),
  90. }, {
  91. .policy = "writeback",
  92. .cr_mask = 0,
  93. .pmd = PMD_SECT_WB,
  94. .pte = L_PTE_MT_WRITEBACK,
  95. .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
  96. }, {
  97. .policy = "writealloc",
  98. .cr_mask = 0,
  99. .pmd = PMD_SECT_WBWA,
  100. .pte = L_PTE_MT_WRITEALLOC,
  101. .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
  102. }
  103. };
  104. #ifdef CONFIG_CPU_CP15
  105. /*
  106. * These are useful for identifying cache coherency
  107. * problems by allowing the cache or the cache and
  108. * writebuffer to be turned off. (Note: the write
  109. * buffer should not be on and the cache off).
  110. */
  111. static int __init early_cachepolicy(char *p)
  112. {
  113. int i;
  114. for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
  115. int len = strlen(cache_policies[i].policy);
  116. if (memcmp(p, cache_policies[i].policy, len) == 0) {
  117. cachepolicy = i;
  118. cr_alignment &= ~cache_policies[i].cr_mask;
  119. cr_no_alignment &= ~cache_policies[i].cr_mask;
  120. break;
  121. }
  122. }
  123. if (i == ARRAY_SIZE(cache_policies))
  124. printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n");
  125. /*
  126. * This restriction is partly to do with the way we boot; it is
  127. * unpredictable to have memory mapped using two different sets of
  128. * memory attributes (shared, type, and cache attribs). We can not
  129. * change these attributes once the initial assembly has setup the
  130. * page tables.
  131. */
  132. if (cpu_architecture() >= CPU_ARCH_ARMv6) {
  133. printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n");
  134. cachepolicy = CPOLICY_WRITEBACK;
  135. }
  136. flush_cache_all();
  137. set_cr(cr_alignment);
  138. return 0;
  139. }
  140. early_param("cachepolicy", early_cachepolicy);
  141. static int __init early_nocache(char *__unused)
  142. {
  143. char *p = "buffered";
  144. printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
  145. early_cachepolicy(p);
  146. return 0;
  147. }
  148. early_param("nocache", early_nocache);
  149. static int __init early_nowrite(char *__unused)
  150. {
  151. char *p = "uncached";
  152. printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
  153. early_cachepolicy(p);
  154. return 0;
  155. }
  156. early_param("nowb", early_nowrite);
  157. #ifndef CONFIG_ARM_LPAE
  158. static int __init early_ecc(char *p)
  159. {
  160. if (memcmp(p, "on", 2) == 0)
  161. ecc_mask = PMD_PROTECTION;
  162. else if (memcmp(p, "off", 3) == 0)
  163. ecc_mask = 0;
  164. return 0;
  165. }
  166. early_param("ecc", early_ecc);
  167. #endif
  168. static int __init noalign_setup(char *__unused)
  169. {
  170. cr_alignment &= ~CR_A;
  171. cr_no_alignment &= ~CR_A;
  172. set_cr(cr_alignment);
  173. return 1;
  174. }
  175. __setup("noalign", noalign_setup);
  176. #ifndef CONFIG_SMP
  177. void adjust_cr(unsigned long mask, unsigned long set)
  178. {
  179. unsigned long flags;
  180. mask &= ~CR_A;
  181. set &= mask;
  182. local_irq_save(flags);
  183. cr_no_alignment = (cr_no_alignment & ~mask) | set;
  184. cr_alignment = (cr_alignment & ~mask) | set;
  185. set_cr((get_cr() & ~mask) | set);
  186. local_irq_restore(flags);
  187. }
  188. #endif
  189. #else /* ifdef CONFIG_CPU_CP15 */
  190. static int __init early_cachepolicy(char *p)
  191. {
  192. pr_warning("cachepolicy kernel parameter not supported without cp15\n");
  193. }
  194. early_param("cachepolicy", early_cachepolicy);
  195. static int __init noalign_setup(char *__unused)
  196. {
  197. pr_warning("noalign kernel parameter not supported without cp15\n");
  198. }
  199. __setup("noalign", noalign_setup);
  200. #endif /* ifdef CONFIG_CPU_CP15 / else */
  201. #define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
  202. #define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE
  203. static struct mem_type mem_types[] = {
  204. [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */
  205. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
  206. L_PTE_SHARED,
  207. .prot_l1 = PMD_TYPE_TABLE,
  208. .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S,
  209. .domain = DOMAIN_IO,
  210. },
  211. [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
  212. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
  213. .prot_l1 = PMD_TYPE_TABLE,
  214. .prot_sect = PROT_SECT_DEVICE,
  215. .domain = DOMAIN_IO,
  216. },
  217. [MT_DEVICE_CACHED] = { /* ioremap_cached */
  218. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
  219. .prot_l1 = PMD_TYPE_TABLE,
  220. .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB,
  221. .domain = DOMAIN_IO,
  222. },
  223. [MT_DEVICE_WC] = { /* ioremap_wc */
  224. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
  225. .prot_l1 = PMD_TYPE_TABLE,
  226. .prot_sect = PROT_SECT_DEVICE,
  227. .domain = DOMAIN_IO,
  228. },
  229. [MT_UNCACHED] = {
  230. .prot_pte = PROT_PTE_DEVICE,
  231. .prot_l1 = PMD_TYPE_TABLE,
  232. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
  233. .domain = DOMAIN_IO,
  234. },
  235. [MT_CACHECLEAN] = {
  236. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
  237. .domain = DOMAIN_KERNEL,
  238. },
  239. #ifndef CONFIG_ARM_LPAE
  240. [MT_MINICLEAN] = {
  241. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
  242. .domain = DOMAIN_KERNEL,
  243. },
  244. #endif
  245. [MT_LOW_VECTORS] = {
  246. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  247. L_PTE_RDONLY,
  248. .prot_l1 = PMD_TYPE_TABLE,
  249. .domain = DOMAIN_USER,
  250. },
  251. [MT_HIGH_VECTORS] = {
  252. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  253. L_PTE_USER | L_PTE_RDONLY,
  254. .prot_l1 = PMD_TYPE_TABLE,
  255. .domain = DOMAIN_USER,
  256. },
  257. [MT_MEMORY] = {
  258. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
  259. .prot_l1 = PMD_TYPE_TABLE,
  260. .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
  261. .domain = DOMAIN_KERNEL,
  262. },
  263. [MT_ROM] = {
  264. .prot_sect = PMD_TYPE_SECT,
  265. .domain = DOMAIN_KERNEL,
  266. },
  267. [MT_MEMORY_NONCACHED] = {
  268. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  269. L_PTE_MT_BUFFERABLE,
  270. .prot_l1 = PMD_TYPE_TABLE,
  271. .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
  272. .domain = DOMAIN_KERNEL,
  273. },
  274. [MT_MEMORY_DTCM] = {
  275. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  276. L_PTE_XN,
  277. .prot_l1 = PMD_TYPE_TABLE,
  278. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
  279. .domain = DOMAIN_KERNEL,
  280. },
  281. [MT_MEMORY_ITCM] = {
  282. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
  283. .prot_l1 = PMD_TYPE_TABLE,
  284. .domain = DOMAIN_KERNEL,
  285. },
  286. [MT_MEMORY_SO] = {
  287. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  288. L_PTE_MT_UNCACHED | L_PTE_XN,
  289. .prot_l1 = PMD_TYPE_TABLE,
  290. .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
  291. PMD_SECT_UNCACHED | PMD_SECT_XN,
  292. .domain = DOMAIN_KERNEL,
  293. },
  294. [MT_MEMORY_DMA_READY] = {
  295. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
  296. .prot_l1 = PMD_TYPE_TABLE,
  297. .domain = DOMAIN_KERNEL,
  298. },
  299. };
  300. const struct mem_type *get_mem_type(unsigned int type)
  301. {
  302. return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
  303. }
  304. EXPORT_SYMBOL(get_mem_type);
  305. /*
  306. * Adjust the PMD section entries according to the CPU in use.
  307. */
  308. static void __init build_mem_type_table(void)
  309. {
  310. struct cachepolicy *cp;
  311. unsigned int cr = get_cr();
  312. pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
  313. pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
  314. int cpu_arch = cpu_architecture();
  315. int i;
  316. if (cpu_arch < CPU_ARCH_ARMv6) {
  317. #if defined(CONFIG_CPU_DCACHE_DISABLE)
  318. if (cachepolicy > CPOLICY_BUFFERED)
  319. cachepolicy = CPOLICY_BUFFERED;
  320. #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
  321. if (cachepolicy > CPOLICY_WRITETHROUGH)
  322. cachepolicy = CPOLICY_WRITETHROUGH;
  323. #endif
  324. }
  325. if (cpu_arch < CPU_ARCH_ARMv5) {
  326. if (cachepolicy >= CPOLICY_WRITEALLOC)
  327. cachepolicy = CPOLICY_WRITEBACK;
  328. ecc_mask = 0;
  329. }
  330. if (is_smp())
  331. cachepolicy = CPOLICY_WRITEALLOC;
  332. /*
  333. * Strip out features not present on earlier architectures.
  334. * Pre-ARMv5 CPUs don't have TEX bits. Pre-ARMv6 CPUs or those
  335. * without extended page tables don't have the 'Shared' bit.
  336. */
  337. if (cpu_arch < CPU_ARCH_ARMv5)
  338. for (i = 0; i < ARRAY_SIZE(mem_types); i++)
  339. mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
  340. if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
  341. for (i = 0; i < ARRAY_SIZE(mem_types); i++)
  342. mem_types[i].prot_sect &= ~PMD_SECT_S;
  343. /*
  344. * ARMv5 and lower, bit 4 must be set for page tables (was: cache
  345. * "update-able on write" bit on ARM610). However, Xscale and
  346. * Xscale3 require this bit to be cleared.
  347. */
  348. if (cpu_is_xscale() || cpu_is_xsc3()) {
  349. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  350. mem_types[i].prot_sect &= ~PMD_BIT4;
  351. mem_types[i].prot_l1 &= ~PMD_BIT4;
  352. }
  353. } else if (cpu_arch < CPU_ARCH_ARMv6) {
  354. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  355. if (mem_types[i].prot_l1)
  356. mem_types[i].prot_l1 |= PMD_BIT4;
  357. if (mem_types[i].prot_sect)
  358. mem_types[i].prot_sect |= PMD_BIT4;
  359. }
  360. }
  361. /*
  362. * Mark the device areas according to the CPU/architecture.
  363. */
  364. if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
  365. if (!cpu_is_xsc3()) {
  366. /*
  367. * Mark device regions on ARMv6+ as execute-never
  368. * to prevent speculative instruction fetches.
  369. */
  370. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
  371. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
  372. mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
  373. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
  374. }
  375. if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
  376. /*
  377. * For ARMv7 with TEX remapping,
  378. * - shared device is SXCB=1100
  379. * - nonshared device is SXCB=0100
  380. * - write combine device mem is SXCB=0001
  381. * (Uncached Normal memory)
  382. */
  383. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
  384. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
  385. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
  386. } else if (cpu_is_xsc3()) {
  387. /*
  388. * For Xscale3,
  389. * - shared device is TEXCB=00101
  390. * - nonshared device is TEXCB=01000
  391. * - write combine device mem is TEXCB=00100
  392. * (Inner/Outer Uncacheable in xsc3 parlance)
  393. */
  394. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
  395. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
  396. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
  397. } else {
  398. /*
  399. * For ARMv6 and ARMv7 without TEX remapping,
  400. * - shared device is TEXCB=00001
  401. * - nonshared device is TEXCB=01000
  402. * - write combine device mem is TEXCB=00100
  403. * (Uncached Normal in ARMv6 parlance).
  404. */
  405. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
  406. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
  407. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
  408. }
  409. } else {
  410. /*
  411. * On others, write combining is "Uncached/Buffered"
  412. */
  413. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
  414. }
  415. /*
  416. * Now deal with the memory-type mappings
  417. */
  418. cp = &cache_policies[cachepolicy];
  419. vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
  420. s2_pgprot = cp->pte_s2;
  421. hyp_device_pgprot = s2_device_pgprot = mem_types[MT_DEVICE].prot_pte;
  422. /*
  423. * ARMv6 and above have extended page tables.
  424. */
  425. if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
  426. #ifndef CONFIG_ARM_LPAE
  427. /*
  428. * Mark cache clean areas and XIP ROM read only
  429. * from SVC mode and no access from userspace.
  430. */
  431. mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
  432. mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
  433. mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
  434. #endif
  435. if (is_smp()) {
  436. /*
  437. * Mark memory with the "shared" attribute
  438. * for SMP systems
  439. */
  440. user_pgprot |= L_PTE_SHARED;
  441. kern_pgprot |= L_PTE_SHARED;
  442. vecs_pgprot |= L_PTE_SHARED;
  443. s2_pgprot |= L_PTE_SHARED;
  444. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
  445. mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
  446. mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
  447. mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
  448. mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
  449. mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
  450. mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
  451. mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
  452. mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
  453. }
  454. }
  455. /*
  456. * Non-cacheable Normal - intended for memory areas that must
  457. * not cause dirty cache line writebacks when used
  458. */
  459. if (cpu_arch >= CPU_ARCH_ARMv6) {
  460. if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
  461. /* Non-cacheable Normal is XCB = 001 */
  462. mem_types[MT_MEMORY_NONCACHED].prot_sect |=
  463. PMD_SECT_BUFFERED;
  464. } else {
  465. /* For both ARMv6 and non-TEX-remapping ARMv7 */
  466. mem_types[MT_MEMORY_NONCACHED].prot_sect |=
  467. PMD_SECT_TEX(1);
  468. }
  469. } else {
  470. mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
  471. }
  472. #ifdef CONFIG_ARM_LPAE
  473. /*
  474. * Do not generate access flag faults for the kernel mappings.
  475. */
  476. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  477. mem_types[i].prot_pte |= PTE_EXT_AF;
  478. if (mem_types[i].prot_sect)
  479. mem_types[i].prot_sect |= PMD_SECT_AF;
  480. }
  481. kern_pgprot |= PTE_EXT_AF;
  482. vecs_pgprot |= PTE_EXT_AF;
  483. #endif
  484. for (i = 0; i < 16; i++) {
  485. pteval_t v = pgprot_val(protection_map[i]);
  486. protection_map[i] = __pgprot(v | user_pgprot);
  487. }
  488. mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
  489. mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
  490. pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
  491. pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
  492. L_PTE_DIRTY | kern_pgprot);
  493. pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
  494. pgprot_s2_device = __pgprot(s2_device_pgprot);
  495. pgprot_hyp_device = __pgprot(hyp_device_pgprot);
  496. mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
  497. mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
  498. mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
  499. mem_types[MT_MEMORY].prot_pte |= kern_pgprot;
  500. mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
  501. mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask;
  502. mem_types[MT_ROM].prot_sect |= cp->pmd;
  503. switch (cp->pmd) {
  504. case PMD_SECT_WT:
  505. mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
  506. break;
  507. case PMD_SECT_WB:
  508. case PMD_SECT_WBWA:
  509. mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
  510. break;
  511. }
  512. printk("Memory policy: ECC %sabled, Data cache %s\n",
  513. ecc_mask ? "en" : "dis", cp->policy);
  514. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  515. struct mem_type *t = &mem_types[i];
  516. if (t->prot_l1)
  517. t->prot_l1 |= PMD_DOMAIN(t->domain);
  518. if (t->prot_sect)
  519. t->prot_sect |= PMD_DOMAIN(t->domain);
  520. }
  521. }
  522. #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
  523. pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
  524. unsigned long size, pgprot_t vma_prot)
  525. {
  526. if (!pfn_valid(pfn))
  527. return pgprot_noncached(vma_prot);
  528. else if (file->f_flags & O_SYNC)
  529. return pgprot_writecombine(vma_prot);
  530. return vma_prot;
  531. }
  532. EXPORT_SYMBOL(phys_mem_access_prot);
  533. #endif
  534. #define vectors_base() (vectors_high() ? 0xffff0000 : 0)
  535. static void __init *early_alloc_aligned(unsigned long sz, unsigned long align)
  536. {
  537. void *ptr = __va(memblock_alloc(sz, align));
  538. memset(ptr, 0, sz);
  539. return ptr;
  540. }
  541. static void __init *early_alloc(unsigned long sz)
  542. {
  543. return early_alloc_aligned(sz, sz);
  544. }
  545. static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
  546. {
  547. if (pmd_none(*pmd)) {
  548. pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
  549. __pmd_populate(pmd, __pa(pte), prot);
  550. }
  551. BUG_ON(pmd_bad(*pmd));
  552. return pte_offset_kernel(pmd, addr);
  553. }
  554. static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
  555. unsigned long end, unsigned long pfn,
  556. const struct mem_type *type)
  557. {
  558. pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1);
  559. do {
  560. set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
  561. pfn++;
  562. } while (pte++, addr += PAGE_SIZE, addr != end);
  563. }
  564. static void __init map_init_section(pmd_t *pmd, unsigned long addr,
  565. unsigned long end, phys_addr_t phys,
  566. const struct mem_type *type)
  567. {
  568. #ifndef CONFIG_ARM_LPAE
  569. /*
  570. * In classic MMU format, puds and pmds are folded in to
  571. * the pgds. pmd_offset gives the PGD entry. PGDs refer to a
  572. * group of L1 entries making up one logical pointer to
  573. * an L2 table (2MB), where as PMDs refer to the individual
  574. * L1 entries (1MB). Hence increment to get the correct
  575. * offset for odd 1MB sections.
  576. * (See arch/arm/include/asm/pgtable-2level.h)
  577. */
  578. if (addr & SECTION_SIZE)
  579. pmd++;
  580. #endif
  581. do {
  582. *pmd = __pmd(phys | type->prot_sect);
  583. phys += SECTION_SIZE;
  584. } while (pmd++, addr += SECTION_SIZE, addr != end);
  585. flush_pmd_entry(pmd);
  586. }
  587. static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
  588. unsigned long end, phys_addr_t phys,
  589. const struct mem_type *type)
  590. {
  591. pmd_t *pmd = pmd_offset(pud, addr);
  592. unsigned long next;
  593. do {
  594. /*
  595. * With LPAE, we must loop over to map
  596. * all the pmds for the given range.
  597. */
  598. next = pmd_addr_end(addr, end);
  599. /*
  600. * Try a section mapping - addr, next and phys must all be
  601. * aligned to a section boundary.
  602. */
  603. if (type->prot_sect &&
  604. ((addr | next | phys) & ~SECTION_MASK) == 0) {
  605. map_init_section(pmd, addr, next, phys, type);
  606. } else {
  607. alloc_init_pte(pmd, addr, next,
  608. __phys_to_pfn(phys), type);
  609. }
  610. phys += next - addr;
  611. } while (pmd++, addr = next, addr != end);
  612. }
  613. static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
  614. unsigned long end, unsigned long phys, const struct mem_type *type)
  615. {
  616. pud_t *pud = pud_offset(pgd, addr);
  617. unsigned long next;
  618. do {
  619. next = pud_addr_end(addr, end);
  620. alloc_init_pmd(pud, addr, next, phys, type);
  621. phys += next - addr;
  622. } while (pud++, addr = next, addr != end);
  623. }
  624. #ifndef CONFIG_ARM_LPAE
  625. static void __init create_36bit_mapping(struct map_desc *md,
  626. const struct mem_type *type)
  627. {
  628. unsigned long addr, length, end;
  629. phys_addr_t phys;
  630. pgd_t *pgd;
  631. addr = md->virtual;
  632. phys = __pfn_to_phys(md->pfn);
  633. length = PAGE_ALIGN(md->length);
  634. if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
  635. printk(KERN_ERR "MM: CPU does not support supersection "
  636. "mapping for 0x%08llx at 0x%08lx\n",
  637. (long long)__pfn_to_phys((u64)md->pfn), addr);
  638. return;
  639. }
  640. /* N.B. ARMv6 supersections are only defined to work with domain 0.
  641. * Since domain assignments can in fact be arbitrary, the
  642. * 'domain == 0' check below is required to insure that ARMv6
  643. * supersections are only allocated for domain 0 regardless
  644. * of the actual domain assignments in use.
  645. */
  646. if (type->domain) {
  647. printk(KERN_ERR "MM: invalid domain in supersection "
  648. "mapping for 0x%08llx at 0x%08lx\n",
  649. (long long)__pfn_to_phys((u64)md->pfn), addr);
  650. return;
  651. }
  652. if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
  653. printk(KERN_ERR "MM: cannot create mapping for 0x%08llx"
  654. " at 0x%08lx invalid alignment\n",
  655. (long long)__pfn_to_phys((u64)md->pfn), addr);
  656. return;
  657. }
  658. /*
  659. * Shift bits [35:32] of address into bits [23:20] of PMD
  660. * (See ARMv6 spec).
  661. */
  662. phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
  663. pgd = pgd_offset_k(addr);
  664. end = addr + length;
  665. do {
  666. pud_t *pud = pud_offset(pgd, addr);
  667. pmd_t *pmd = pmd_offset(pud, addr);
  668. int i;
  669. for (i = 0; i < 16; i++)
  670. *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER);
  671. addr += SUPERSECTION_SIZE;
  672. phys += SUPERSECTION_SIZE;
  673. pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
  674. } while (addr != end);
  675. }
  676. #endif /* !CONFIG_ARM_LPAE */
  677. /*
  678. * Create the page directory entries and any necessary
  679. * page tables for the mapping specified by `md'. We
  680. * are able to cope here with varying sizes and address
  681. * offsets, and we take full advantage of sections and
  682. * supersections.
  683. */
  684. static void __init create_mapping(struct map_desc *md)
  685. {
  686. unsigned long addr, length, end;
  687. phys_addr_t phys;
  688. const struct mem_type *type;
  689. pgd_t *pgd;
  690. if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
  691. printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx"
  692. " at 0x%08lx in user region\n",
  693. (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
  694. return;
  695. }
  696. if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
  697. md->virtual >= PAGE_OFFSET &&
  698. (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
  699. printk(KERN_WARNING "BUG: mapping for 0x%08llx"
  700. " at 0x%08lx out of vmalloc space\n",
  701. (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
  702. }
  703. type = &mem_types[md->type];
  704. #ifndef CONFIG_ARM_LPAE
  705. /*
  706. * Catch 36-bit addresses
  707. */
  708. if (md->pfn >= 0x100000) {
  709. create_36bit_mapping(md, type);
  710. return;
  711. }
  712. #endif
  713. addr = md->virtual & PAGE_MASK;
  714. phys = __pfn_to_phys(md->pfn);
  715. length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
  716. if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
  717. printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not "
  718. "be mapped using pages, ignoring.\n",
  719. (long long)__pfn_to_phys(md->pfn), addr);
  720. return;
  721. }
  722. pgd = pgd_offset_k(addr);
  723. end = addr + length;
  724. do {
  725. unsigned long next = pgd_addr_end(addr, end);
  726. alloc_init_pud(pgd, addr, next, phys, type);
  727. phys += next - addr;
  728. addr = next;
  729. } while (pgd++, addr != end);
  730. }
  731. /*
  732. * Create the architecture specific mappings
  733. */
  734. void __init iotable_init(struct map_desc *io_desc, int nr)
  735. {
  736. struct map_desc *md;
  737. struct vm_struct *vm;
  738. struct static_vm *svm;
  739. if (!nr)
  740. return;
  741. svm = early_alloc_aligned(sizeof(*svm) * nr, __alignof__(*svm));
  742. for (md = io_desc; nr; md++, nr--) {
  743. create_mapping(md);
  744. vm = &svm->vm;
  745. vm->addr = (void *)(md->virtual & PAGE_MASK);
  746. vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
  747. vm->phys_addr = __pfn_to_phys(md->pfn);
  748. vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;
  749. vm->flags |= VM_ARM_MTYPE(md->type);
  750. vm->caller = iotable_init;
  751. add_static_vm_early(svm++);
  752. }
  753. }
  754. void __init vm_reserve_area_early(unsigned long addr, unsigned long size,
  755. void *caller)
  756. {
  757. struct vm_struct *vm;
  758. struct static_vm *svm;
  759. svm = early_alloc_aligned(sizeof(*svm), __alignof__(*svm));
  760. vm = &svm->vm;
  761. vm->addr = (void *)addr;
  762. vm->size = size;
  763. vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;
  764. vm->caller = caller;
  765. add_static_vm_early(svm);
  766. }
  767. #ifndef CONFIG_ARM_LPAE
  768. /*
  769. * The Linux PMD is made of two consecutive section entries covering 2MB
  770. * (see definition in include/asm/pgtable-2level.h). However a call to
  771. * create_mapping() may optimize static mappings by using individual
  772. * 1MB section mappings. This leaves the actual PMD potentially half
  773. * initialized if the top or bottom section entry isn't used, leaving it
  774. * open to problems if a subsequent ioremap() or vmalloc() tries to use
  775. * the virtual space left free by that unused section entry.
  776. *
  777. * Let's avoid the issue by inserting dummy vm entries covering the unused
  778. * PMD halves once the static mappings are in place.
  779. */
  780. static void __init pmd_empty_section_gap(unsigned long addr)
  781. {
  782. vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap);
  783. }
  784. static void __init fill_pmd_gaps(void)
  785. {
  786. struct static_vm *svm;
  787. struct vm_struct *vm;
  788. unsigned long addr, next = 0;
  789. pmd_t *pmd;
  790. list_for_each_entry(svm, &static_vmlist, list) {
  791. vm = &svm->vm;
  792. addr = (unsigned long)vm->addr;
  793. if (addr < next)
  794. continue;
  795. /*
  796. * Check if this vm starts on an odd section boundary.
  797. * If so and the first section entry for this PMD is free
  798. * then we block the corresponding virtual address.
  799. */
  800. if ((addr & ~PMD_MASK) == SECTION_SIZE) {
  801. pmd = pmd_off_k(addr);
  802. if (pmd_none(*pmd))
  803. pmd_empty_section_gap(addr & PMD_MASK);
  804. }
  805. /*
  806. * Then check if this vm ends on an odd section boundary.
  807. * If so and the second section entry for this PMD is empty
  808. * then we block the corresponding virtual address.
  809. */
  810. addr += vm->size;
  811. if ((addr & ~PMD_MASK) == SECTION_SIZE) {
  812. pmd = pmd_off_k(addr) + 1;
  813. if (pmd_none(*pmd))
  814. pmd_empty_section_gap(addr);
  815. }
  816. /* no need to look at any vm entry until we hit the next PMD */
  817. next = (addr + PMD_SIZE - 1) & PMD_MASK;
  818. }
  819. }
  820. #else
  821. #define fill_pmd_gaps() do { } while (0)
  822. #endif
  823. #if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H)
  824. static void __init pci_reserve_io(void)
  825. {
  826. struct static_vm *svm;
  827. svm = find_static_vm_vaddr((void *)PCI_IO_VIRT_BASE);
  828. if (svm)
  829. return;
  830. vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io);
  831. }
  832. #else
  833. #define pci_reserve_io() do { } while (0)
  834. #endif
  835. #ifdef CONFIG_DEBUG_LL
  836. void __init debug_ll_io_init(void)
  837. {
  838. struct map_desc map;
  839. debug_ll_addr(&map.pfn, &map.virtual);
  840. if (!map.pfn || !map.virtual)
  841. return;
  842. map.pfn = __phys_to_pfn(map.pfn);
  843. map.virtual &= PAGE_MASK;
  844. map.length = PAGE_SIZE;
  845. map.type = MT_DEVICE;
  846. create_mapping(&map);
  847. }
  848. #endif
  849. static void * __initdata vmalloc_min =
  850. (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET);
  851. /*
  852. * vmalloc=size forces the vmalloc area to be exactly 'size'
  853. * bytes. This can be used to increase (or decrease) the vmalloc
  854. * area - the default is 240m.
  855. */
  856. static int __init early_vmalloc(char *arg)
  857. {
  858. unsigned long vmalloc_reserve = memparse(arg, NULL);
  859. if (vmalloc_reserve < SZ_16M) {
  860. vmalloc_reserve = SZ_16M;
  861. printk(KERN_WARNING
  862. "vmalloc area too small, limiting to %luMB\n",
  863. vmalloc_reserve >> 20);
  864. }
  865. if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
  866. vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
  867. printk(KERN_WARNING
  868. "vmalloc area is too big, limiting to %luMB\n",
  869. vmalloc_reserve >> 20);
  870. }
  871. vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
  872. return 0;
  873. }
  874. early_param("vmalloc", early_vmalloc);
  875. phys_addr_t arm_lowmem_limit __initdata = 0;
  876. void __init sanity_check_meminfo(void)
  877. {
  878. int i, j, highmem = 0;
  879. for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
  880. struct membank *bank = &meminfo.bank[j];
  881. *bank = meminfo.bank[i];
  882. if (bank->start > ULONG_MAX)
  883. highmem = 1;
  884. #ifdef CONFIG_HIGHMEM
  885. if (__va(bank->start) >= vmalloc_min ||
  886. __va(bank->start) < (void *)PAGE_OFFSET)
  887. highmem = 1;
  888. bank->highmem = highmem;
  889. /*
  890. * Split those memory banks which are partially overlapping
  891. * the vmalloc area greatly simplifying things later.
  892. */
  893. if (!highmem && __va(bank->start) < vmalloc_min &&
  894. bank->size > vmalloc_min - __va(bank->start)) {
  895. if (meminfo.nr_banks >= NR_BANKS) {
  896. printk(KERN_CRIT "NR_BANKS too low, "
  897. "ignoring high memory\n");
  898. } else {
  899. memmove(bank + 1, bank,
  900. (meminfo.nr_banks - i) * sizeof(*bank));
  901. meminfo.nr_banks++;
  902. i++;
  903. bank[1].size -= vmalloc_min - __va(bank->start);
  904. bank[1].start = __pa(vmalloc_min - 1) + 1;
  905. bank[1].highmem = highmem = 1;
  906. j++;
  907. }
  908. bank->size = vmalloc_min - __va(bank->start);
  909. }
  910. #else
  911. bank->highmem = highmem;
  912. /*
  913. * Highmem banks not allowed with !CONFIG_HIGHMEM.
  914. */
  915. if (highmem) {
  916. printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
  917. "(!CONFIG_HIGHMEM).\n",
  918. (unsigned long long)bank->start,
  919. (unsigned long long)bank->start + bank->size - 1);
  920. continue;
  921. }
  922. /*
  923. * Check whether this memory bank would entirely overlap
  924. * the vmalloc area.
  925. */
  926. if (__va(bank->start) >= vmalloc_min ||
  927. __va(bank->start) < (void *)PAGE_OFFSET) {
  928. printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
  929. "(vmalloc region overlap).\n",
  930. (unsigned long long)bank->start,
  931. (unsigned long long)bank->start + bank->size - 1);
  932. continue;
  933. }
  934. /*
  935. * Check whether this memory bank would partially overlap
  936. * the vmalloc area.
  937. */
  938. if (__va(bank->start + bank->size - 1) >= vmalloc_min ||
  939. __va(bank->start + bank->size - 1) <= __va(bank->start)) {
  940. unsigned long newsize = vmalloc_min - __va(bank->start);
  941. printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx "
  942. "to -%.8llx (vmalloc region overlap).\n",
  943. (unsigned long long)bank->start,
  944. (unsigned long long)bank->start + bank->size - 1,
  945. (unsigned long long)bank->start + newsize - 1);
  946. bank->size = newsize;
  947. }
  948. #endif
  949. if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit)
  950. arm_lowmem_limit = bank->start + bank->size;
  951. j++;
  952. }
  953. #ifdef CONFIG_HIGHMEM
  954. if (highmem) {
  955. const char *reason = NULL;
  956. if (cache_is_vipt_aliasing()) {
  957. /*
  958. * Interactions between kmap and other mappings
  959. * make highmem support with aliasing VIPT caches
  960. * rather difficult.
  961. */
  962. reason = "with VIPT aliasing cache";
  963. }
  964. if (reason) {
  965. printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
  966. reason);
  967. while (j > 0 && meminfo.bank[j - 1].highmem)
  968. j--;
  969. }
  970. }
  971. #endif
  972. meminfo.nr_banks = j;
  973. high_memory = __va(arm_lowmem_limit - 1) + 1;
  974. memblock_set_current_limit(arm_lowmem_limit);
  975. }
  976. static inline void prepare_page_table(void)
  977. {
  978. unsigned long addr;
  979. phys_addr_t end;
  980. /*
  981. * Clear out all the mappings below the kernel image.
  982. */
  983. for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
  984. pmd_clear(pmd_off_k(addr));
  985. #ifdef CONFIG_XIP_KERNEL
  986. /* The XIP kernel is mapped in the module area -- skip over it */
  987. addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK;
  988. #endif
  989. for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE)
  990. pmd_clear(pmd_off_k(addr));
  991. /*
  992. * Find the end of the first block of lowmem.
  993. */
  994. end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
  995. if (end >= arm_lowmem_limit)
  996. end = arm_lowmem_limit;
  997. /*
  998. * Clear out all the kernel space mappings, except for the first
  999. * memory bank, up to the vmalloc region.
  1000. */
  1001. for (addr = __phys_to_virt(end);
  1002. addr < VMALLOC_START; addr += PMD_SIZE)
  1003. pmd_clear(pmd_off_k(addr));
  1004. }
  1005. #ifdef CONFIG_ARM_LPAE
  1006. /* the first page is reserved for pgd */
  1007. #define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
  1008. PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
  1009. #else
  1010. #define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
  1011. #endif
  1012. /*
  1013. * Reserve the special regions of memory
  1014. */
  1015. void __init arm_mm_memblock_reserve(void)
  1016. {
  1017. /*
  1018. * Reserve the page tables. These are already in use,
  1019. * and can only be in node 0.
  1020. */
  1021. memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE);
  1022. #ifdef CONFIG_SA1111
  1023. /*
  1024. * Because of the SA1111 DMA bug, we want to preserve our
  1025. * precious DMA-able memory...
  1026. */
  1027. memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
  1028. #endif
  1029. }
  1030. /*
  1031. * Set up the device mappings. Since we clear out the page tables for all
  1032. * mappings above VMALLOC_START, we will remove any debug device mappings.
  1033. * This means you have to be careful how you debug this function, or any
  1034. * called function. This means you can't use any function or debugging
  1035. * method which may touch any device, otherwise the kernel _will_ crash.
  1036. */
  1037. static void __init devicemaps_init(struct machine_desc *mdesc)
  1038. {
  1039. struct map_desc map;
  1040. unsigned long addr;
  1041. void *vectors;
  1042. /*
  1043. * Allocate the vector page early.
  1044. */
  1045. vectors = early_alloc(PAGE_SIZE);
  1046. early_trap_init(vectors);
  1047. for (addr = VMALLOC_START; addr; addr += PMD_SIZE)
  1048. pmd_clear(pmd_off_k(addr));
  1049. /*
  1050. * Map the kernel if it is XIP.
  1051. * It is always first in the modulearea.
  1052. */
  1053. #ifdef CONFIG_XIP_KERNEL
  1054. map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
  1055. map.virtual = MODULES_VADDR;
  1056. map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
  1057. map.type = MT_ROM;
  1058. create_mapping(&map);
  1059. #endif
  1060. /*
  1061. * Map the cache flushing regions.
  1062. */
  1063. #ifdef FLUSH_BASE
  1064. map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
  1065. map.virtual = FLUSH_BASE;
  1066. map.length = SZ_1M;
  1067. map.type = MT_CACHECLEAN;
  1068. create_mapping(&map);
  1069. #endif
  1070. #ifdef FLUSH_BASE_MINICACHE
  1071. map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
  1072. map.virtual = FLUSH_BASE_MINICACHE;
  1073. map.length = SZ_1M;
  1074. map.type = MT_MINICLEAN;
  1075. create_mapping(&map);
  1076. #endif
  1077. /*
  1078. * Create a mapping for the machine vectors at the high-vectors
  1079. * location (0xffff0000). If we aren't using high-vectors, also
  1080. * create a mapping at the low-vectors virtual address.
  1081. */
  1082. map.pfn = __phys_to_pfn(virt_to_phys(vectors));
  1083. map.virtual = 0xffff0000;
  1084. map.length = PAGE_SIZE;
  1085. map.type = MT_HIGH_VECTORS;
  1086. create_mapping(&map);
  1087. if (!vectors_high()) {
  1088. map.virtual = 0;
  1089. map.type = MT_LOW_VECTORS;
  1090. create_mapping(&map);
  1091. }
  1092. /*
  1093. * Ask the machine support to map in the statically mapped devices.
  1094. */
  1095. if (mdesc->map_io)
  1096. mdesc->map_io();
  1097. fill_pmd_gaps();
  1098. /* Reserve fixed i/o space in VMALLOC region */
  1099. pci_reserve_io();
  1100. /*
  1101. * Finally flush the caches and tlb to ensure that we're in a
  1102. * consistent state wrt the writebuffer. This also ensures that
  1103. * any write-allocated cache lines in the vector page are written
  1104. * back. After this point, we can start to touch devices again.
  1105. */
  1106. local_flush_tlb_all();
  1107. flush_cache_all();
  1108. }
  1109. static void __init kmap_init(void)
  1110. {
  1111. #ifdef CONFIG_HIGHMEM
  1112. pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
  1113. PKMAP_BASE, _PAGE_KERNEL_TABLE);
  1114. #endif
  1115. }
  1116. static void __init map_lowmem(void)
  1117. {
  1118. struct memblock_region *reg;
  1119. /* Map all the lowmem memory banks. */
  1120. for_each_memblock(memory, reg) {
  1121. phys_addr_t start = reg->base;
  1122. phys_addr_t end = start + reg->size;
  1123. struct map_desc map;
  1124. if (end > arm_lowmem_limit)
  1125. end = arm_lowmem_limit;
  1126. if (start >= end)
  1127. break;
  1128. map.pfn = __phys_to_pfn(start);
  1129. map.virtual = __phys_to_virt(start);
  1130. map.length = end - start;
  1131. map.type = MT_MEMORY;
  1132. create_mapping(&map);
  1133. }
  1134. }
  1135. /*
  1136. * paging_init() sets up the page tables, initialises the zone memory
  1137. * maps, and sets up the zero page, bad page and bad page tables.
  1138. */
  1139. void __init paging_init(struct machine_desc *mdesc)
  1140. {
  1141. void *zero_page;
  1142. memblock_set_current_limit(arm_lowmem_limit);
  1143. build_mem_type_table();
  1144. prepare_page_table();
  1145. map_lowmem();
  1146. dma_contiguous_remap();
  1147. devicemaps_init(mdesc);
  1148. kmap_init();
  1149. tcm_init();
  1150. top_pmd = pmd_off_k(0xffff0000);
  1151. /* allocate the zero page. */
  1152. zero_page = early_alloc(PAGE_SIZE);
  1153. bootmem_init();
  1154. empty_zero_page = virt_to_page(zero_page);
  1155. __flush_dcache_page(NULL, empty_zero_page);
  1156. }