mmu.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425
  1. /*
  2. * linux/arch/arm/mm/mmu.c
  3. *
  4. * Copyright (C) 1995-2005 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #include <linux/module.h>
  11. #include <linux/kernel.h>
  12. #include <linux/errno.h>
  13. #include <linux/init.h>
  14. #include <linux/mman.h>
  15. #include <linux/nodemask.h>
  16. #include <linux/memblock.h>
  17. #include <linux/fs.h>
  18. #include <linux/vmalloc.h>
  19. #include <linux/sizes.h>
  20. #include <asm/cp15.h>
  21. #include <asm/cputype.h>
  22. #include <asm/sections.h>
  23. #include <asm/cachetype.h>
  24. #include <asm/setup.h>
  25. #include <asm/smp_plat.h>
  26. #include <asm/tlb.h>
  27. #include <asm/highmem.h>
  28. #include <asm/system_info.h>
  29. #include <asm/traps.h>
  30. #include <asm/procinfo.h>
  31. #include <asm/memory.h>
  32. #include <asm/mach/arch.h>
  33. #include <asm/mach/map.h>
  34. #include <asm/mach/pci.h>
  35. #include "mm.h"
  36. #include "tcm.h"
  37. /*
  38. * empty_zero_page is a special page that is used for
  39. * zero-initialized data and COW.
  40. */
  41. struct page *empty_zero_page;
  42. EXPORT_SYMBOL(empty_zero_page);
  43. /*
  44. * The pmd table for the upper-most set of pages.
  45. */
  46. pmd_t *top_pmd;
  47. #define CPOLICY_UNCACHED 0
  48. #define CPOLICY_BUFFERED 1
  49. #define CPOLICY_WRITETHROUGH 2
  50. #define CPOLICY_WRITEBACK 3
  51. #define CPOLICY_WRITEALLOC 4
  52. static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
  53. static unsigned int ecc_mask __initdata = 0;
  54. pgprot_t pgprot_user;
  55. pgprot_t pgprot_kernel;
  56. pgprot_t pgprot_hyp_device;
  57. pgprot_t pgprot_s2;
  58. pgprot_t pgprot_s2_device;
  59. EXPORT_SYMBOL(pgprot_user);
  60. EXPORT_SYMBOL(pgprot_kernel);
  61. struct cachepolicy {
  62. const char policy[16];
  63. unsigned int cr_mask;
  64. pmdval_t pmd;
  65. pteval_t pte;
  66. pteval_t pte_s2;
  67. };
  68. #ifdef CONFIG_ARM_LPAE
  69. #define s2_policy(policy) policy
  70. #else
  71. #define s2_policy(policy) 0
  72. #endif
  73. static struct cachepolicy cache_policies[] __initdata = {
  74. {
  75. .policy = "uncached",
  76. .cr_mask = CR_W|CR_C,
  77. .pmd = PMD_SECT_UNCACHED,
  78. .pte = L_PTE_MT_UNCACHED,
  79. .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
  80. }, {
  81. .policy = "buffered",
  82. .cr_mask = CR_C,
  83. .pmd = PMD_SECT_BUFFERED,
  84. .pte = L_PTE_MT_BUFFERABLE,
  85. .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
  86. }, {
  87. .policy = "writethrough",
  88. .cr_mask = 0,
  89. .pmd = PMD_SECT_WT,
  90. .pte = L_PTE_MT_WRITETHROUGH,
  91. .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH),
  92. }, {
  93. .policy = "writeback",
  94. .cr_mask = 0,
  95. .pmd = PMD_SECT_WB,
  96. .pte = L_PTE_MT_WRITEBACK,
  97. .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
  98. }, {
  99. .policy = "writealloc",
  100. .cr_mask = 0,
  101. .pmd = PMD_SECT_WBWA,
  102. .pte = L_PTE_MT_WRITEALLOC,
  103. .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
  104. }
  105. };
  106. #ifdef CONFIG_CPU_CP15
  107. /*
  108. * These are useful for identifying cache coherency
  109. * problems by allowing the cache or the cache and
  110. * writebuffer to be turned off. (Note: the write
  111. * buffer should not be on and the cache off).
  112. */
  113. static int __init early_cachepolicy(char *p)
  114. {
  115. int i;
  116. for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
  117. int len = strlen(cache_policies[i].policy);
  118. if (memcmp(p, cache_policies[i].policy, len) == 0) {
  119. cachepolicy = i;
  120. cr_alignment &= ~cache_policies[i].cr_mask;
  121. cr_no_alignment &= ~cache_policies[i].cr_mask;
  122. break;
  123. }
  124. }
  125. if (i == ARRAY_SIZE(cache_policies))
  126. printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n");
  127. /*
  128. * This restriction is partly to do with the way we boot; it is
  129. * unpredictable to have memory mapped using two different sets of
  130. * memory attributes (shared, type, and cache attribs). We can not
  131. * change these attributes once the initial assembly has setup the
  132. * page tables.
  133. */
  134. if (cpu_architecture() >= CPU_ARCH_ARMv6) {
  135. printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n");
  136. cachepolicy = CPOLICY_WRITEBACK;
  137. }
  138. flush_cache_all();
  139. set_cr(cr_alignment);
  140. return 0;
  141. }
  142. early_param("cachepolicy", early_cachepolicy);
  143. static int __init early_nocache(char *__unused)
  144. {
  145. char *p = "buffered";
  146. printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
  147. early_cachepolicy(p);
  148. return 0;
  149. }
  150. early_param("nocache", early_nocache);
  151. static int __init early_nowrite(char *__unused)
  152. {
  153. char *p = "uncached";
  154. printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
  155. early_cachepolicy(p);
  156. return 0;
  157. }
  158. early_param("nowb", early_nowrite);
  159. #ifndef CONFIG_ARM_LPAE
  160. static int __init early_ecc(char *p)
  161. {
  162. if (memcmp(p, "on", 2) == 0)
  163. ecc_mask = PMD_PROTECTION;
  164. else if (memcmp(p, "off", 3) == 0)
  165. ecc_mask = 0;
  166. return 0;
  167. }
  168. early_param("ecc", early_ecc);
  169. #endif
  170. static int __init noalign_setup(char *__unused)
  171. {
  172. cr_alignment &= ~CR_A;
  173. cr_no_alignment &= ~CR_A;
  174. set_cr(cr_alignment);
  175. return 1;
  176. }
  177. __setup("noalign", noalign_setup);
  178. #ifndef CONFIG_SMP
  179. void adjust_cr(unsigned long mask, unsigned long set)
  180. {
  181. unsigned long flags;
  182. mask &= ~CR_A;
  183. set &= mask;
  184. local_irq_save(flags);
  185. cr_no_alignment = (cr_no_alignment & ~mask) | set;
  186. cr_alignment = (cr_alignment & ~mask) | set;
  187. set_cr((get_cr() & ~mask) | set);
  188. local_irq_restore(flags);
  189. }
  190. #endif
  191. #else /* ifdef CONFIG_CPU_CP15 */
  192. static int __init early_cachepolicy(char *p)
  193. {
  194. pr_warning("cachepolicy kernel parameter not supported without cp15\n");
  195. }
  196. early_param("cachepolicy", early_cachepolicy);
  197. static int __init noalign_setup(char *__unused)
  198. {
  199. pr_warning("noalign kernel parameter not supported without cp15\n");
  200. }
  201. __setup("noalign", noalign_setup);
  202. #endif /* ifdef CONFIG_CPU_CP15 / else */
  203. #define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
  204. #define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE
  205. static struct mem_type mem_types[] = {
  206. [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */
  207. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
  208. L_PTE_SHARED,
  209. .prot_l1 = PMD_TYPE_TABLE,
  210. .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S,
  211. .domain = DOMAIN_IO,
  212. },
  213. [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
  214. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
  215. .prot_l1 = PMD_TYPE_TABLE,
  216. .prot_sect = PROT_SECT_DEVICE,
  217. .domain = DOMAIN_IO,
  218. },
  219. [MT_DEVICE_CACHED] = { /* ioremap_cached */
  220. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
  221. .prot_l1 = PMD_TYPE_TABLE,
  222. .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB,
  223. .domain = DOMAIN_IO,
  224. },
  225. [MT_DEVICE_WC] = { /* ioremap_wc */
  226. .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
  227. .prot_l1 = PMD_TYPE_TABLE,
  228. .prot_sect = PROT_SECT_DEVICE,
  229. .domain = DOMAIN_IO,
  230. },
  231. [MT_UNCACHED] = {
  232. .prot_pte = PROT_PTE_DEVICE,
  233. .prot_l1 = PMD_TYPE_TABLE,
  234. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
  235. .domain = DOMAIN_IO,
  236. },
  237. [MT_CACHECLEAN] = {
  238. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
  239. .domain = DOMAIN_KERNEL,
  240. },
  241. #ifndef CONFIG_ARM_LPAE
  242. [MT_MINICLEAN] = {
  243. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
  244. .domain = DOMAIN_KERNEL,
  245. },
  246. #endif
  247. [MT_LOW_VECTORS] = {
  248. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  249. L_PTE_RDONLY,
  250. .prot_l1 = PMD_TYPE_TABLE,
  251. .domain = DOMAIN_USER,
  252. },
  253. [MT_HIGH_VECTORS] = {
  254. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  255. L_PTE_USER | L_PTE_RDONLY,
  256. .prot_l1 = PMD_TYPE_TABLE,
  257. .domain = DOMAIN_USER,
  258. },
  259. [MT_MEMORY] = {
  260. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
  261. .prot_l1 = PMD_TYPE_TABLE,
  262. .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
  263. .domain = DOMAIN_KERNEL,
  264. },
  265. [MT_ROM] = {
  266. .prot_sect = PMD_TYPE_SECT,
  267. .domain = DOMAIN_KERNEL,
  268. },
  269. [MT_MEMORY_NONCACHED] = {
  270. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  271. L_PTE_MT_BUFFERABLE,
  272. .prot_l1 = PMD_TYPE_TABLE,
  273. .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
  274. .domain = DOMAIN_KERNEL,
  275. },
  276. [MT_MEMORY_DTCM] = {
  277. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  278. L_PTE_XN,
  279. .prot_l1 = PMD_TYPE_TABLE,
  280. .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
  281. .domain = DOMAIN_KERNEL,
  282. },
  283. [MT_MEMORY_ITCM] = {
  284. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
  285. .prot_l1 = PMD_TYPE_TABLE,
  286. .domain = DOMAIN_KERNEL,
  287. },
  288. [MT_MEMORY_SO] = {
  289. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
  290. L_PTE_MT_UNCACHED | L_PTE_XN,
  291. .prot_l1 = PMD_TYPE_TABLE,
  292. .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
  293. PMD_SECT_UNCACHED | PMD_SECT_XN,
  294. .domain = DOMAIN_KERNEL,
  295. },
  296. [MT_MEMORY_DMA_READY] = {
  297. .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
  298. .prot_l1 = PMD_TYPE_TABLE,
  299. .domain = DOMAIN_KERNEL,
  300. },
  301. };
  302. const struct mem_type *get_mem_type(unsigned int type)
  303. {
  304. return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
  305. }
  306. EXPORT_SYMBOL(get_mem_type);
  307. /*
  308. * Adjust the PMD section entries according to the CPU in use.
  309. */
  310. static void __init build_mem_type_table(void)
  311. {
  312. struct cachepolicy *cp;
  313. unsigned int cr = get_cr();
  314. pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
  315. pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
  316. int cpu_arch = cpu_architecture();
  317. int i;
  318. if (cpu_arch < CPU_ARCH_ARMv6) {
  319. #if defined(CONFIG_CPU_DCACHE_DISABLE)
  320. if (cachepolicy > CPOLICY_BUFFERED)
  321. cachepolicy = CPOLICY_BUFFERED;
  322. #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
  323. if (cachepolicy > CPOLICY_WRITETHROUGH)
  324. cachepolicy = CPOLICY_WRITETHROUGH;
  325. #endif
  326. }
  327. if (cpu_arch < CPU_ARCH_ARMv5) {
  328. if (cachepolicy >= CPOLICY_WRITEALLOC)
  329. cachepolicy = CPOLICY_WRITEBACK;
  330. ecc_mask = 0;
  331. }
  332. if (is_smp())
  333. cachepolicy = CPOLICY_WRITEALLOC;
  334. /*
  335. * Strip out features not present on earlier architectures.
  336. * Pre-ARMv5 CPUs don't have TEX bits. Pre-ARMv6 CPUs or those
  337. * without extended page tables don't have the 'Shared' bit.
  338. */
  339. if (cpu_arch < CPU_ARCH_ARMv5)
  340. for (i = 0; i < ARRAY_SIZE(mem_types); i++)
  341. mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
  342. if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
  343. for (i = 0; i < ARRAY_SIZE(mem_types); i++)
  344. mem_types[i].prot_sect &= ~PMD_SECT_S;
  345. /*
  346. * ARMv5 and lower, bit 4 must be set for page tables (was: cache
  347. * "update-able on write" bit on ARM610). However, Xscale and
  348. * Xscale3 require this bit to be cleared.
  349. */
  350. if (cpu_is_xscale() || cpu_is_xsc3()) {
  351. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  352. mem_types[i].prot_sect &= ~PMD_BIT4;
  353. mem_types[i].prot_l1 &= ~PMD_BIT4;
  354. }
  355. } else if (cpu_arch < CPU_ARCH_ARMv6) {
  356. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  357. if (mem_types[i].prot_l1)
  358. mem_types[i].prot_l1 |= PMD_BIT4;
  359. if (mem_types[i].prot_sect)
  360. mem_types[i].prot_sect |= PMD_BIT4;
  361. }
  362. }
  363. /*
  364. * Mark the device areas according to the CPU/architecture.
  365. */
  366. if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
  367. if (!cpu_is_xsc3()) {
  368. /*
  369. * Mark device regions on ARMv6+ as execute-never
  370. * to prevent speculative instruction fetches.
  371. */
  372. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
  373. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
  374. mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
  375. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
  376. }
  377. if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
  378. /*
  379. * For ARMv7 with TEX remapping,
  380. * - shared device is SXCB=1100
  381. * - nonshared device is SXCB=0100
  382. * - write combine device mem is SXCB=0001
  383. * (Uncached Normal memory)
  384. */
  385. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
  386. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
  387. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
  388. } else if (cpu_is_xsc3()) {
  389. /*
  390. * For Xscale3,
  391. * - shared device is TEXCB=00101
  392. * - nonshared device is TEXCB=01000
  393. * - write combine device mem is TEXCB=00100
  394. * (Inner/Outer Uncacheable in xsc3 parlance)
  395. */
  396. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
  397. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
  398. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
  399. } else {
  400. /*
  401. * For ARMv6 and ARMv7 without TEX remapping,
  402. * - shared device is TEXCB=00001
  403. * - nonshared device is TEXCB=01000
  404. * - write combine device mem is TEXCB=00100
  405. * (Uncached Normal in ARMv6 parlance).
  406. */
  407. mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
  408. mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
  409. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
  410. }
  411. } else {
  412. /*
  413. * On others, write combining is "Uncached/Buffered"
  414. */
  415. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
  416. }
  417. /*
  418. * Now deal with the memory-type mappings
  419. */
  420. cp = &cache_policies[cachepolicy];
  421. vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
  422. s2_pgprot = cp->pte_s2;
  423. hyp_device_pgprot = s2_device_pgprot = mem_types[MT_DEVICE].prot_pte;
  424. /*
  425. * ARMv6 and above have extended page tables.
  426. */
  427. if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
  428. #ifndef CONFIG_ARM_LPAE
  429. /*
  430. * Mark cache clean areas and XIP ROM read only
  431. * from SVC mode and no access from userspace.
  432. */
  433. mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
  434. mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
  435. mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
  436. #endif
  437. if (is_smp()) {
  438. /*
  439. * Mark memory with the "shared" attribute
  440. * for SMP systems
  441. */
  442. user_pgprot |= L_PTE_SHARED;
  443. kern_pgprot |= L_PTE_SHARED;
  444. vecs_pgprot |= L_PTE_SHARED;
  445. s2_pgprot |= L_PTE_SHARED;
  446. mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
  447. mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
  448. mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
  449. mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
  450. mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
  451. mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
  452. mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
  453. mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
  454. mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
  455. }
  456. }
  457. /*
  458. * Non-cacheable Normal - intended for memory areas that must
  459. * not cause dirty cache line writebacks when used
  460. */
  461. if (cpu_arch >= CPU_ARCH_ARMv6) {
  462. if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
  463. /* Non-cacheable Normal is XCB = 001 */
  464. mem_types[MT_MEMORY_NONCACHED].prot_sect |=
  465. PMD_SECT_BUFFERED;
  466. } else {
  467. /* For both ARMv6 and non-TEX-remapping ARMv7 */
  468. mem_types[MT_MEMORY_NONCACHED].prot_sect |=
  469. PMD_SECT_TEX(1);
  470. }
  471. } else {
  472. mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
  473. }
  474. #ifdef CONFIG_ARM_LPAE
  475. /*
  476. * Do not generate access flag faults for the kernel mappings.
  477. */
  478. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  479. mem_types[i].prot_pte |= PTE_EXT_AF;
  480. if (mem_types[i].prot_sect)
  481. mem_types[i].prot_sect |= PMD_SECT_AF;
  482. }
  483. kern_pgprot |= PTE_EXT_AF;
  484. vecs_pgprot |= PTE_EXT_AF;
  485. #endif
  486. for (i = 0; i < 16; i++) {
  487. pteval_t v = pgprot_val(protection_map[i]);
  488. protection_map[i] = __pgprot(v | user_pgprot);
  489. }
  490. mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
  491. mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
  492. pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
  493. pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
  494. L_PTE_DIRTY | kern_pgprot);
  495. pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
  496. pgprot_s2_device = __pgprot(s2_device_pgprot);
  497. pgprot_hyp_device = __pgprot(hyp_device_pgprot);
  498. mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
  499. mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
  500. mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
  501. mem_types[MT_MEMORY].prot_pte |= kern_pgprot;
  502. mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
  503. mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask;
  504. mem_types[MT_ROM].prot_sect |= cp->pmd;
  505. switch (cp->pmd) {
  506. case PMD_SECT_WT:
  507. mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
  508. break;
  509. case PMD_SECT_WB:
  510. case PMD_SECT_WBWA:
  511. mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
  512. break;
  513. }
  514. pr_info("Memory policy: %sData cache %s\n",
  515. ecc_mask ? "ECC enabled, " : "", cp->policy);
  516. for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
  517. struct mem_type *t = &mem_types[i];
  518. if (t->prot_l1)
  519. t->prot_l1 |= PMD_DOMAIN(t->domain);
  520. if (t->prot_sect)
  521. t->prot_sect |= PMD_DOMAIN(t->domain);
  522. }
  523. }
  524. #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
  525. pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
  526. unsigned long size, pgprot_t vma_prot)
  527. {
  528. if (!pfn_valid(pfn))
  529. return pgprot_noncached(vma_prot);
  530. else if (file->f_flags & O_SYNC)
  531. return pgprot_writecombine(vma_prot);
  532. return vma_prot;
  533. }
  534. EXPORT_SYMBOL(phys_mem_access_prot);
  535. #endif
  536. #define vectors_base() (vectors_high() ? 0xffff0000 : 0)
  537. static void __init *early_alloc_aligned(unsigned long sz, unsigned long align)
  538. {
  539. void *ptr = __va(memblock_alloc(sz, align));
  540. memset(ptr, 0, sz);
  541. return ptr;
  542. }
  543. static void __init *early_alloc(unsigned long sz)
  544. {
  545. return early_alloc_aligned(sz, sz);
  546. }
  547. static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
  548. {
  549. if (pmd_none(*pmd)) {
  550. pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
  551. __pmd_populate(pmd, __pa(pte), prot);
  552. }
  553. BUG_ON(pmd_bad(*pmd));
  554. return pte_offset_kernel(pmd, addr);
  555. }
  556. static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
  557. unsigned long end, unsigned long pfn,
  558. const struct mem_type *type)
  559. {
  560. pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1);
  561. do {
  562. set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
  563. pfn++;
  564. } while (pte++, addr += PAGE_SIZE, addr != end);
  565. }
  566. static void __init __map_init_section(pmd_t *pmd, unsigned long addr,
  567. unsigned long end, phys_addr_t phys,
  568. const struct mem_type *type)
  569. {
  570. pmd_t *p = pmd;
  571. #ifndef CONFIG_ARM_LPAE
  572. /*
  573. * In classic MMU format, puds and pmds are folded in to
  574. * the pgds. pmd_offset gives the PGD entry. PGDs refer to a
  575. * group of L1 entries making up one logical pointer to
  576. * an L2 table (2MB), where as PMDs refer to the individual
  577. * L1 entries (1MB). Hence increment to get the correct
  578. * offset for odd 1MB sections.
  579. * (See arch/arm/include/asm/pgtable-2level.h)
  580. */
  581. if (addr & SECTION_SIZE)
  582. pmd++;
  583. #endif
  584. do {
  585. *pmd = __pmd(phys | type->prot_sect);
  586. phys += SECTION_SIZE;
  587. } while (pmd++, addr += SECTION_SIZE, addr != end);
  588. flush_pmd_entry(p);
  589. }
  590. static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
  591. unsigned long end, phys_addr_t phys,
  592. const struct mem_type *type)
  593. {
  594. pmd_t *pmd = pmd_offset(pud, addr);
  595. unsigned long next;
  596. do {
  597. /*
  598. * With LPAE, we must loop over to map
  599. * all the pmds for the given range.
  600. */
  601. next = pmd_addr_end(addr, end);
  602. /*
  603. * Try a section mapping - addr, next and phys must all be
  604. * aligned to a section boundary.
  605. */
  606. if (type->prot_sect &&
  607. ((addr | next | phys) & ~SECTION_MASK) == 0) {
  608. __map_init_section(pmd, addr, next, phys, type);
  609. } else {
  610. alloc_init_pte(pmd, addr, next,
  611. __phys_to_pfn(phys), type);
  612. }
  613. phys += next - addr;
  614. } while (pmd++, addr = next, addr != end);
  615. }
  616. static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
  617. unsigned long end, phys_addr_t phys,
  618. const struct mem_type *type)
  619. {
  620. pud_t *pud = pud_offset(pgd, addr);
  621. unsigned long next;
  622. do {
  623. next = pud_addr_end(addr, end);
  624. alloc_init_pmd(pud, addr, next, phys, type);
  625. phys += next - addr;
  626. } while (pud++, addr = next, addr != end);
  627. }
  628. #ifndef CONFIG_ARM_LPAE
  629. static void __init create_36bit_mapping(struct map_desc *md,
  630. const struct mem_type *type)
  631. {
  632. unsigned long addr, length, end;
  633. phys_addr_t phys;
  634. pgd_t *pgd;
  635. addr = md->virtual;
  636. phys = __pfn_to_phys(md->pfn);
  637. length = PAGE_ALIGN(md->length);
  638. if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
  639. printk(KERN_ERR "MM: CPU does not support supersection "
  640. "mapping for 0x%08llx at 0x%08lx\n",
  641. (long long)__pfn_to_phys((u64)md->pfn), addr);
  642. return;
  643. }
  644. /* N.B. ARMv6 supersections are only defined to work with domain 0.
  645. * Since domain assignments can in fact be arbitrary, the
  646. * 'domain == 0' check below is required to insure that ARMv6
  647. * supersections are only allocated for domain 0 regardless
  648. * of the actual domain assignments in use.
  649. */
  650. if (type->domain) {
  651. printk(KERN_ERR "MM: invalid domain in supersection "
  652. "mapping for 0x%08llx at 0x%08lx\n",
  653. (long long)__pfn_to_phys((u64)md->pfn), addr);
  654. return;
  655. }
  656. if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
  657. printk(KERN_ERR "MM: cannot create mapping for 0x%08llx"
  658. " at 0x%08lx invalid alignment\n",
  659. (long long)__pfn_to_phys((u64)md->pfn), addr);
  660. return;
  661. }
  662. /*
  663. * Shift bits [35:32] of address into bits [23:20] of PMD
  664. * (See ARMv6 spec).
  665. */
  666. phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
  667. pgd = pgd_offset_k(addr);
  668. end = addr + length;
  669. do {
  670. pud_t *pud = pud_offset(pgd, addr);
  671. pmd_t *pmd = pmd_offset(pud, addr);
  672. int i;
  673. for (i = 0; i < 16; i++)
  674. *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER);
  675. addr += SUPERSECTION_SIZE;
  676. phys += SUPERSECTION_SIZE;
  677. pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
  678. } while (addr != end);
  679. }
  680. #endif /* !CONFIG_ARM_LPAE */
  681. /*
  682. * Create the page directory entries and any necessary
  683. * page tables for the mapping specified by `md'. We
  684. * are able to cope here with varying sizes and address
  685. * offsets, and we take full advantage of sections and
  686. * supersections.
  687. */
  688. static void __init create_mapping(struct map_desc *md)
  689. {
  690. unsigned long addr, length, end;
  691. phys_addr_t phys;
  692. const struct mem_type *type;
  693. pgd_t *pgd;
  694. if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
  695. printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx"
  696. " at 0x%08lx in user region\n",
  697. (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
  698. return;
  699. }
  700. if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
  701. md->virtual >= PAGE_OFFSET &&
  702. (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
  703. printk(KERN_WARNING "BUG: mapping for 0x%08llx"
  704. " at 0x%08lx out of vmalloc space\n",
  705. (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
  706. }
  707. type = &mem_types[md->type];
  708. #ifndef CONFIG_ARM_LPAE
  709. /*
  710. * Catch 36-bit addresses
  711. */
  712. if (md->pfn >= 0x100000) {
  713. create_36bit_mapping(md, type);
  714. return;
  715. }
  716. #endif
  717. addr = md->virtual & PAGE_MASK;
  718. phys = __pfn_to_phys(md->pfn);
  719. length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
  720. if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
  721. printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not "
  722. "be mapped using pages, ignoring.\n",
  723. (long long)__pfn_to_phys(md->pfn), addr);
  724. return;
  725. }
  726. pgd = pgd_offset_k(addr);
  727. end = addr + length;
  728. do {
  729. unsigned long next = pgd_addr_end(addr, end);
  730. alloc_init_pud(pgd, addr, next, phys, type);
  731. phys += next - addr;
  732. addr = next;
  733. } while (pgd++, addr != end);
  734. }
  735. /*
  736. * Create the architecture specific mappings
  737. */
  738. void __init iotable_init(struct map_desc *io_desc, int nr)
  739. {
  740. struct map_desc *md;
  741. struct vm_struct *vm;
  742. struct static_vm *svm;
  743. if (!nr)
  744. return;
  745. svm = early_alloc_aligned(sizeof(*svm) * nr, __alignof__(*svm));
  746. for (md = io_desc; nr; md++, nr--) {
  747. create_mapping(md);
  748. vm = &svm->vm;
  749. vm->addr = (void *)(md->virtual & PAGE_MASK);
  750. vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
  751. vm->phys_addr = __pfn_to_phys(md->pfn);
  752. vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;
  753. vm->flags |= VM_ARM_MTYPE(md->type);
  754. vm->caller = iotable_init;
  755. add_static_vm_early(svm++);
  756. }
  757. }
  758. void __init vm_reserve_area_early(unsigned long addr, unsigned long size,
  759. void *caller)
  760. {
  761. struct vm_struct *vm;
  762. struct static_vm *svm;
  763. svm = early_alloc_aligned(sizeof(*svm), __alignof__(*svm));
  764. vm = &svm->vm;
  765. vm->addr = (void *)addr;
  766. vm->size = size;
  767. vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;
  768. vm->caller = caller;
  769. add_static_vm_early(svm);
  770. }
  771. #ifndef CONFIG_ARM_LPAE
  772. /*
  773. * The Linux PMD is made of two consecutive section entries covering 2MB
  774. * (see definition in include/asm/pgtable-2level.h). However a call to
  775. * create_mapping() may optimize static mappings by using individual
  776. * 1MB section mappings. This leaves the actual PMD potentially half
  777. * initialized if the top or bottom section entry isn't used, leaving it
  778. * open to problems if a subsequent ioremap() or vmalloc() tries to use
  779. * the virtual space left free by that unused section entry.
  780. *
  781. * Let's avoid the issue by inserting dummy vm entries covering the unused
  782. * PMD halves once the static mappings are in place.
  783. */
  784. static void __init pmd_empty_section_gap(unsigned long addr)
  785. {
  786. vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap);
  787. }
  788. static void __init fill_pmd_gaps(void)
  789. {
  790. struct static_vm *svm;
  791. struct vm_struct *vm;
  792. unsigned long addr, next = 0;
  793. pmd_t *pmd;
  794. list_for_each_entry(svm, &static_vmlist, list) {
  795. vm = &svm->vm;
  796. addr = (unsigned long)vm->addr;
  797. if (addr < next)
  798. continue;
  799. /*
  800. * Check if this vm starts on an odd section boundary.
  801. * If so and the first section entry for this PMD is free
  802. * then we block the corresponding virtual address.
  803. */
  804. if ((addr & ~PMD_MASK) == SECTION_SIZE) {
  805. pmd = pmd_off_k(addr);
  806. if (pmd_none(*pmd))
  807. pmd_empty_section_gap(addr & PMD_MASK);
  808. }
  809. /*
  810. * Then check if this vm ends on an odd section boundary.
  811. * If so and the second section entry for this PMD is empty
  812. * then we block the corresponding virtual address.
  813. */
  814. addr += vm->size;
  815. if ((addr & ~PMD_MASK) == SECTION_SIZE) {
  816. pmd = pmd_off_k(addr) + 1;
  817. if (pmd_none(*pmd))
  818. pmd_empty_section_gap(addr);
  819. }
  820. /* no need to look at any vm entry until we hit the next PMD */
  821. next = (addr + PMD_SIZE - 1) & PMD_MASK;
  822. }
  823. }
  824. #else
  825. #define fill_pmd_gaps() do { } while (0)
  826. #endif
  827. #if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H)
  828. static void __init pci_reserve_io(void)
  829. {
  830. struct static_vm *svm;
  831. svm = find_static_vm_vaddr((void *)PCI_IO_VIRT_BASE);
  832. if (svm)
  833. return;
  834. vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io);
  835. }
  836. #else
  837. #define pci_reserve_io() do { } while (0)
  838. #endif
  839. #ifdef CONFIG_DEBUG_LL
  840. void __init debug_ll_io_init(void)
  841. {
  842. struct map_desc map;
  843. debug_ll_addr(&map.pfn, &map.virtual);
  844. if (!map.pfn || !map.virtual)
  845. return;
  846. map.pfn = __phys_to_pfn(map.pfn);
  847. map.virtual &= PAGE_MASK;
  848. map.length = PAGE_SIZE;
  849. map.type = MT_DEVICE;
  850. iotable_init(&map, 1);
  851. }
  852. #endif
  853. static void * __initdata vmalloc_min =
  854. (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET);
  855. /*
  856. * vmalloc=size forces the vmalloc area to be exactly 'size'
  857. * bytes. This can be used to increase (or decrease) the vmalloc
  858. * area - the default is 240m.
  859. */
  860. static int __init early_vmalloc(char *arg)
  861. {
  862. unsigned long vmalloc_reserve = memparse(arg, NULL);
  863. if (vmalloc_reserve < SZ_16M) {
  864. vmalloc_reserve = SZ_16M;
  865. printk(KERN_WARNING
  866. "vmalloc area too small, limiting to %luMB\n",
  867. vmalloc_reserve >> 20);
  868. }
  869. if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
  870. vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
  871. printk(KERN_WARNING
  872. "vmalloc area is too big, limiting to %luMB\n",
  873. vmalloc_reserve >> 20);
  874. }
  875. vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
  876. return 0;
  877. }
  878. early_param("vmalloc", early_vmalloc);
  879. phys_addr_t arm_lowmem_limit __initdata = 0;
  880. void __init sanity_check_meminfo(void)
  881. {
  882. phys_addr_t memblock_limit = 0;
  883. int i, j, highmem = 0;
  884. phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1;
  885. for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
  886. struct membank *bank = &meminfo.bank[j];
  887. phys_addr_t size_limit;
  888. *bank = meminfo.bank[i];
  889. size_limit = bank->size;
  890. if (bank->start >= vmalloc_limit)
  891. highmem = 1;
  892. else
  893. size_limit = vmalloc_limit - bank->start;
  894. bank->highmem = highmem;
  895. #ifdef CONFIG_HIGHMEM
  896. /*
  897. * Split those memory banks which are partially overlapping
  898. * the vmalloc area greatly simplifying things later.
  899. */
  900. if (!highmem && bank->size > size_limit) {
  901. if (meminfo.nr_banks >= NR_BANKS) {
  902. printk(KERN_CRIT "NR_BANKS too low, "
  903. "ignoring high memory\n");
  904. } else {
  905. memmove(bank + 1, bank,
  906. (meminfo.nr_banks - i) * sizeof(*bank));
  907. meminfo.nr_banks++;
  908. i++;
  909. bank[1].size -= size_limit;
  910. bank[1].start = vmalloc_limit;
  911. bank[1].highmem = highmem = 1;
  912. j++;
  913. }
  914. bank->size = size_limit;
  915. }
  916. #else
  917. /*
  918. * Highmem banks not allowed with !CONFIG_HIGHMEM.
  919. */
  920. if (highmem) {
  921. printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
  922. "(!CONFIG_HIGHMEM).\n",
  923. (unsigned long long)bank->start,
  924. (unsigned long long)bank->start + bank->size - 1);
  925. continue;
  926. }
  927. /*
  928. * Check whether this memory bank would partially overlap
  929. * the vmalloc area.
  930. */
  931. if (bank->size > size_limit) {
  932. printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx "
  933. "to -%.8llx (vmalloc region overlap).\n",
  934. (unsigned long long)bank->start,
  935. (unsigned long long)bank->start + bank->size - 1,
  936. (unsigned long long)bank->start + size_limit - 1);
  937. bank->size = size_limit;
  938. }
  939. #endif
  940. if (!bank->highmem) {
  941. phys_addr_t bank_end = bank->start + bank->size;
  942. if (bank_end > arm_lowmem_limit)
  943. arm_lowmem_limit = bank_end;
  944. /*
  945. * Find the first non-section-aligned page, and point
  946. * memblock_limit at it. This relies on rounding the
  947. * limit down to be section-aligned, which happens at
  948. * the end of this function.
  949. *
  950. * With this algorithm, the start or end of almost any
  951. * bank can be non-section-aligned. The only exception
  952. * is that the start of the bank 0 must be section-
  953. * aligned, since otherwise memory would need to be
  954. * allocated when mapping the start of bank 0, which
  955. * occurs before any free memory is mapped.
  956. */
  957. if (!memblock_limit) {
  958. if (!IS_ALIGNED(bank->start, SECTION_SIZE))
  959. memblock_limit = bank->start;
  960. else if (!IS_ALIGNED(bank_end, SECTION_SIZE))
  961. memblock_limit = bank_end;
  962. }
  963. }
  964. j++;
  965. }
  966. #ifdef CONFIG_HIGHMEM
  967. if (highmem) {
  968. const char *reason = NULL;
  969. if (cache_is_vipt_aliasing()) {
  970. /*
  971. * Interactions between kmap and other mappings
  972. * make highmem support with aliasing VIPT caches
  973. * rather difficult.
  974. */
  975. reason = "with VIPT aliasing cache";
  976. }
  977. if (reason) {
  978. printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
  979. reason);
  980. while (j > 0 && meminfo.bank[j - 1].highmem)
  981. j--;
  982. }
  983. }
  984. #endif
  985. meminfo.nr_banks = j;
  986. high_memory = __va(arm_lowmem_limit - 1) + 1;
  987. /*
  988. * Round the memblock limit down to a section size. This
  989. * helps to ensure that we will allocate memory from the
  990. * last full section, which should be mapped.
  991. */
  992. if (memblock_limit)
  993. memblock_limit = round_down(memblock_limit, SECTION_SIZE);
  994. if (!memblock_limit)
  995. memblock_limit = arm_lowmem_limit;
  996. memblock_set_current_limit(memblock_limit);
  997. }
  998. static inline void prepare_page_table(void)
  999. {
  1000. unsigned long addr;
  1001. phys_addr_t end;
  1002. /*
  1003. * Clear out all the mappings below the kernel image.
  1004. */
  1005. for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
  1006. pmd_clear(pmd_off_k(addr));
  1007. #ifdef CONFIG_XIP_KERNEL
  1008. /* The XIP kernel is mapped in the module area -- skip over it */
  1009. addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK;
  1010. #endif
  1011. for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE)
  1012. pmd_clear(pmd_off_k(addr));
  1013. /*
  1014. * Find the end of the first block of lowmem.
  1015. */
  1016. end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
  1017. if (end >= arm_lowmem_limit)
  1018. end = arm_lowmem_limit;
  1019. /*
  1020. * Clear out all the kernel space mappings, except for the first
  1021. * memory bank, up to the vmalloc region.
  1022. */
  1023. for (addr = __phys_to_virt(end);
  1024. addr < VMALLOC_START; addr += PMD_SIZE)
  1025. pmd_clear(pmd_off_k(addr));
  1026. }
  1027. #ifdef CONFIG_ARM_LPAE
  1028. /* the first page is reserved for pgd */
  1029. #define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
  1030. PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
  1031. #else
  1032. #define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
  1033. #endif
  1034. /*
  1035. * Reserve the special regions of memory
  1036. */
  1037. void __init arm_mm_memblock_reserve(void)
  1038. {
  1039. /*
  1040. * Reserve the page tables. These are already in use,
  1041. * and can only be in node 0.
  1042. */
  1043. memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE);
  1044. #ifdef CONFIG_SA1111
  1045. /*
  1046. * Because of the SA1111 DMA bug, we want to preserve our
  1047. * precious DMA-able memory...
  1048. */
  1049. memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
  1050. #endif
  1051. }
  1052. /*
  1053. * Set up the device mappings. Since we clear out the page tables for all
  1054. * mappings above VMALLOC_START, we will remove any debug device mappings.
  1055. * This means you have to be careful how you debug this function, or any
  1056. * called function. This means you can't use any function or debugging
  1057. * method which may touch any device, otherwise the kernel _will_ crash.
  1058. */
  1059. static void __init devicemaps_init(const struct machine_desc *mdesc)
  1060. {
  1061. struct map_desc map;
  1062. unsigned long addr;
  1063. void *vectors;
  1064. /*
  1065. * Allocate the vector page early.
  1066. */
  1067. vectors = early_alloc(PAGE_SIZE * 2);
  1068. early_trap_init(vectors);
  1069. for (addr = VMALLOC_START; addr; addr += PMD_SIZE)
  1070. pmd_clear(pmd_off_k(addr));
  1071. /*
  1072. * Map the kernel if it is XIP.
  1073. * It is always first in the modulearea.
  1074. */
  1075. #ifdef CONFIG_XIP_KERNEL
  1076. map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
  1077. map.virtual = MODULES_VADDR;
  1078. map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
  1079. map.type = MT_ROM;
  1080. create_mapping(&map);
  1081. #endif
  1082. /*
  1083. * Map the cache flushing regions.
  1084. */
  1085. #ifdef FLUSH_BASE
  1086. map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
  1087. map.virtual = FLUSH_BASE;
  1088. map.length = SZ_1M;
  1089. map.type = MT_CACHECLEAN;
  1090. create_mapping(&map);
  1091. #endif
  1092. #ifdef FLUSH_BASE_MINICACHE
  1093. map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
  1094. map.virtual = FLUSH_BASE_MINICACHE;
  1095. map.length = SZ_1M;
  1096. map.type = MT_MINICLEAN;
  1097. create_mapping(&map);
  1098. #endif
  1099. /*
  1100. * Create a mapping for the machine vectors at the high-vectors
  1101. * location (0xffff0000). If we aren't using high-vectors, also
  1102. * create a mapping at the low-vectors virtual address.
  1103. */
  1104. map.pfn = __phys_to_pfn(virt_to_phys(vectors));
  1105. map.virtual = 0xffff0000;
  1106. map.length = PAGE_SIZE;
  1107. #ifdef CONFIG_KUSER_HELPERS
  1108. map.type = MT_HIGH_VECTORS;
  1109. #else
  1110. map.type = MT_LOW_VECTORS;
  1111. #endif
  1112. create_mapping(&map);
  1113. if (!vectors_high()) {
  1114. map.virtual = 0;
  1115. map.length = PAGE_SIZE * 2;
  1116. map.type = MT_LOW_VECTORS;
  1117. create_mapping(&map);
  1118. }
  1119. /* Now create a kernel read-only mapping */
  1120. map.pfn += 1;
  1121. map.virtual = 0xffff0000 + PAGE_SIZE;
  1122. map.length = PAGE_SIZE;
  1123. map.type = MT_LOW_VECTORS;
  1124. create_mapping(&map);
  1125. /*
  1126. * Ask the machine support to map in the statically mapped devices.
  1127. */
  1128. if (mdesc->map_io)
  1129. mdesc->map_io();
  1130. else
  1131. debug_ll_io_init();
  1132. fill_pmd_gaps();
  1133. /* Reserve fixed i/o space in VMALLOC region */
  1134. pci_reserve_io();
  1135. /*
  1136. * Finally flush the caches and tlb to ensure that we're in a
  1137. * consistent state wrt the writebuffer. This also ensures that
  1138. * any write-allocated cache lines in the vector page are written
  1139. * back. After this point, we can start to touch devices again.
  1140. */
  1141. local_flush_tlb_all();
  1142. flush_cache_all();
  1143. }
  1144. static void __init kmap_init(void)
  1145. {
  1146. #ifdef CONFIG_HIGHMEM
  1147. pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
  1148. PKMAP_BASE, _PAGE_KERNEL_TABLE);
  1149. #endif
  1150. }
  1151. static void __init map_lowmem(void)
  1152. {
  1153. struct memblock_region *reg;
  1154. /* Map all the lowmem memory banks. */
  1155. for_each_memblock(memory, reg) {
  1156. phys_addr_t start = reg->base;
  1157. phys_addr_t end = start + reg->size;
  1158. struct map_desc map;
  1159. if (end > arm_lowmem_limit)
  1160. end = arm_lowmem_limit;
  1161. if (start >= end)
  1162. break;
  1163. map.pfn = __phys_to_pfn(start);
  1164. map.virtual = __phys_to_virt(start);
  1165. map.length = end - start;
  1166. map.type = MT_MEMORY;
  1167. create_mapping(&map);
  1168. }
  1169. }
  1170. #ifdef CONFIG_ARM_LPAE
  1171. /*
  1172. * early_paging_init() recreates boot time page table setup, allowing machines
  1173. * to switch over to a high (>4G) address space on LPAE systems
  1174. */
  1175. void __init early_paging_init(const struct machine_desc *mdesc,
  1176. struct proc_info_list *procinfo)
  1177. {
  1178. pmdval_t pmdprot = procinfo->__cpu_mm_mmu_flags;
  1179. unsigned long map_start, map_end;
  1180. pgd_t *pgd0, *pgdk;
  1181. pud_t *pud0, *pudk, *pud_start;
  1182. pmd_t *pmd0, *pmdk;
  1183. phys_addr_t phys;
  1184. int i;
  1185. if (!(mdesc->init_meminfo))
  1186. return;
  1187. /* remap kernel code and data */
  1188. map_start = init_mm.start_code;
  1189. map_end = init_mm.brk;
  1190. /* get a handle on things... */
  1191. pgd0 = pgd_offset_k(0);
  1192. pud_start = pud0 = pud_offset(pgd0, 0);
  1193. pmd0 = pmd_offset(pud0, 0);
  1194. pgdk = pgd_offset_k(map_start);
  1195. pudk = pud_offset(pgdk, map_start);
  1196. pmdk = pmd_offset(pudk, map_start);
  1197. mdesc->init_meminfo();
  1198. /* Run the patch stub to update the constants */
  1199. fixup_pv_table(&__pv_table_begin,
  1200. (&__pv_table_end - &__pv_table_begin) << 2);
  1201. /*
  1202. * Cache cleaning operations for self-modifying code
  1203. * We should clean the entries by MVA but running a
  1204. * for loop over every pv_table entry pointer would
  1205. * just complicate the code.
  1206. */
  1207. flush_cache_louis();
  1208. dsb();
  1209. isb();
  1210. /* remap level 1 table */
  1211. for (i = 0; i < PTRS_PER_PGD; pud0++, i++) {
  1212. set_pud(pud0,
  1213. __pud(__pa(pmd0) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
  1214. pmd0 += PTRS_PER_PMD;
  1215. }
  1216. /* remap pmds for kernel mapping */
  1217. phys = __pa(map_start) & PMD_MASK;
  1218. do {
  1219. *pmdk++ = __pmd(phys | pmdprot);
  1220. phys += PMD_SIZE;
  1221. } while (phys < map_end);
  1222. flush_cache_all();
  1223. cpu_switch_mm(pgd0, &init_mm);
  1224. cpu_set_ttbr(1, __pa(pgd0) + TTBR1_OFFSET);
  1225. local_flush_bp_all();
  1226. local_flush_tlb_all();
  1227. }
  1228. #else
  1229. void __init early_paging_init(const struct machine_desc *mdesc,
  1230. struct proc_info_list *procinfo)
  1231. {
  1232. if (mdesc->init_meminfo)
  1233. mdesc->init_meminfo();
  1234. }
  1235. #endif
  1236. /*
  1237. * paging_init() sets up the page tables, initialises the zone memory
  1238. * maps, and sets up the zero page, bad page and bad page tables.
  1239. */
  1240. void __init paging_init(const struct machine_desc *mdesc)
  1241. {
  1242. void *zero_page;
  1243. build_mem_type_table();
  1244. prepare_page_table();
  1245. map_lowmem();
  1246. dma_contiguous_remap();
  1247. devicemaps_init(mdesc);
  1248. kmap_init();
  1249. tcm_init();
  1250. top_pmd = pmd_off_k(0xffff0000);
  1251. /* allocate the zero page. */
  1252. zero_page = early_alloc(PAGE_SIZE);
  1253. bootmem_init();
  1254. empty_zero_page = virt_to_page(zero_page);
  1255. __flush_dcache_page(NULL, empty_zero_page);
  1256. }