setup.c 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610
  1. /*
  2. * Copyright 2010 Tilera Corporation. All Rights Reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public License
  6. * as published by the Free Software Foundation, version 2.
  7. *
  8. * This program is distributed in the hope that it will be useful, but
  9. * WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  11. * NON INFRINGEMENT. See the GNU General Public License for
  12. * more details.
  13. */
  14. #include <linux/sched.h>
  15. #include <linux/kernel.h>
  16. #include <linux/mmzone.h>
  17. #include <linux/bootmem.h>
  18. #include <linux/module.h>
  19. #include <linux/node.h>
  20. #include <linux/cpu.h>
  21. #include <linux/ioport.h>
  22. #include <linux/irq.h>
  23. #include <linux/kexec.h>
  24. #include <linux/pci.h>
  25. #include <linux/initrd.h>
  26. #include <linux/io.h>
  27. #include <linux/highmem.h>
  28. #include <linux/smp.h>
  29. #include <linux/timex.h>
  30. #include <linux/hugetlb.h>
  31. #include <linux/start_kernel.h>
  32. #include <asm/setup.h>
  33. #include <asm/sections.h>
  34. #include <asm/cacheflush.h>
  35. #include <asm/pgalloc.h>
  36. #include <asm/mmu_context.h>
  37. #include <hv/hypervisor.h>
  38. #include <arch/interrupts.h>
  39. /* <linux/smp.h> doesn't provide this definition. */
  40. #ifndef CONFIG_SMP
  41. #define setup_max_cpus 1
  42. #endif
  43. static inline int ABS(int x) { return x >= 0 ? x : -x; }
  44. /* Chip information */
  45. char chip_model[64] __write_once;
  46. struct pglist_data node_data[MAX_NUMNODES] __read_mostly;
  47. EXPORT_SYMBOL(node_data);
  48. /* Information on the NUMA nodes that we compute early */
  49. unsigned long __cpuinitdata node_start_pfn[MAX_NUMNODES];
  50. unsigned long __cpuinitdata node_end_pfn[MAX_NUMNODES];
  51. unsigned long __initdata node_memmap_pfn[MAX_NUMNODES];
  52. unsigned long __initdata node_percpu_pfn[MAX_NUMNODES];
  53. unsigned long __initdata node_free_pfn[MAX_NUMNODES];
  54. static unsigned long __initdata node_percpu[MAX_NUMNODES];
  55. /*
  56. * per-CPU stack and boot info.
  57. */
  58. DEFINE_PER_CPU(unsigned long, boot_sp) =
  59. (unsigned long)init_stack + THREAD_SIZE;
  60. #ifdef CONFIG_SMP
  61. DEFINE_PER_CPU(unsigned long, boot_pc) = (unsigned long)start_kernel;
  62. #else
  63. /*
  64. * The variable must be __initdata since it references __init code.
  65. * With CONFIG_SMP it is per-cpu data, which is exempt from validation.
  66. */
  67. unsigned long __initdata boot_pc = (unsigned long)start_kernel;
  68. #endif
  69. #ifdef CONFIG_HIGHMEM
  70. /* Page frame index of end of lowmem on each controller. */
  71. unsigned long __cpuinitdata node_lowmem_end_pfn[MAX_NUMNODES];
  72. /* Number of pages that can be mapped into lowmem. */
  73. static unsigned long __initdata mappable_physpages;
  74. #endif
  75. /* Data on which physical memory controller corresponds to which NUMA node */
  76. int node_controller[MAX_NUMNODES] = { [0 ... MAX_NUMNODES-1] = -1 };
  77. #ifdef CONFIG_HIGHMEM
  78. /* Map information from VAs to PAs */
  79. unsigned long pbase_map[1 << (32 - HPAGE_SHIFT)]
  80. __write_once __attribute__((aligned(L2_CACHE_BYTES)));
  81. EXPORT_SYMBOL(pbase_map);
  82. /* Map information from PAs to VAs */
  83. void *vbase_map[NR_PA_HIGHBIT_VALUES]
  84. __write_once __attribute__((aligned(L2_CACHE_BYTES)));
  85. EXPORT_SYMBOL(vbase_map);
  86. #endif
  87. /* Node number as a function of the high PA bits */
  88. int highbits_to_node[NR_PA_HIGHBIT_VALUES] __write_once;
  89. EXPORT_SYMBOL(highbits_to_node);
  90. static unsigned int __initdata maxmem_pfn = -1U;
  91. static unsigned int __initdata maxnodemem_pfn[MAX_NUMNODES] = {
  92. [0 ... MAX_NUMNODES-1] = -1U
  93. };
  94. static nodemask_t __initdata isolnodes;
  95. #ifdef CONFIG_PCI
  96. enum { DEFAULT_PCI_RESERVE_MB = 64 };
  97. static unsigned int __initdata pci_reserve_mb = DEFAULT_PCI_RESERVE_MB;
  98. unsigned long __initdata pci_reserve_start_pfn = -1U;
  99. unsigned long __initdata pci_reserve_end_pfn = -1U;
  100. #endif
  101. static int __init setup_maxmem(char *str)
  102. {
  103. unsigned long long maxmem;
  104. if (str == NULL || (maxmem = memparse(str, NULL)) == 0)
  105. return -EINVAL;
  106. maxmem_pfn = (maxmem >> HPAGE_SHIFT) << (HPAGE_SHIFT - PAGE_SHIFT);
  107. pr_info("Forcing RAM used to no more than %dMB\n",
  108. maxmem_pfn >> (20 - PAGE_SHIFT));
  109. return 0;
  110. }
  111. early_param("maxmem", setup_maxmem);
  112. static int __init setup_maxnodemem(char *str)
  113. {
  114. char *endp;
  115. unsigned long long maxnodemem;
  116. long node;
  117. node = str ? simple_strtoul(str, &endp, 0) : INT_MAX;
  118. if (node >= MAX_NUMNODES || *endp != ':')
  119. return -EINVAL;
  120. maxnodemem = memparse(endp+1, NULL);
  121. maxnodemem_pfn[node] = (maxnodemem >> HPAGE_SHIFT) <<
  122. (HPAGE_SHIFT - PAGE_SHIFT);
  123. pr_info("Forcing RAM used on node %ld to no more than %dMB\n",
  124. node, maxnodemem_pfn[node] >> (20 - PAGE_SHIFT));
  125. return 0;
  126. }
  127. early_param("maxnodemem", setup_maxnodemem);
  128. static int __init setup_isolnodes(char *str)
  129. {
  130. char buf[MAX_NUMNODES * 5];
  131. if (str == NULL || nodelist_parse(str, isolnodes) != 0)
  132. return -EINVAL;
  133. nodelist_scnprintf(buf, sizeof(buf), isolnodes);
  134. pr_info("Set isolnodes value to '%s'\n", buf);
  135. return 0;
  136. }
  137. early_param("isolnodes", setup_isolnodes);
  138. #ifdef CONFIG_PCI
  139. static int __init setup_pci_reserve(char* str)
  140. {
  141. unsigned long mb;
  142. if (str == NULL || strict_strtoul(str, 0, &mb) != 0 ||
  143. mb > 3 * 1024)
  144. return -EINVAL;
  145. pci_reserve_mb = mb;
  146. pr_info("Reserving %dMB for PCIE root complex mappings\n",
  147. pci_reserve_mb);
  148. return 0;
  149. }
  150. early_param("pci_reserve", setup_pci_reserve);
  151. #endif
  152. #ifndef __tilegx__
  153. /*
  154. * vmalloc=size forces the vmalloc area to be exactly 'size' bytes.
  155. * This can be used to increase (or decrease) the vmalloc area.
  156. */
  157. static int __init parse_vmalloc(char *arg)
  158. {
  159. if (!arg)
  160. return -EINVAL;
  161. VMALLOC_RESERVE = (memparse(arg, &arg) + PGDIR_SIZE - 1) & PGDIR_MASK;
  162. /* See validate_va() for more on this test. */
  163. if ((long)_VMALLOC_START >= 0)
  164. early_panic("\"vmalloc=%#lx\" value too large: maximum %#lx\n",
  165. VMALLOC_RESERVE, _VMALLOC_END - 0x80000000UL);
  166. return 0;
  167. }
  168. early_param("vmalloc", parse_vmalloc);
  169. #endif
  170. #ifdef CONFIG_HIGHMEM
  171. /*
  172. * Determine for each controller where its lowmem is mapped and how much of
  173. * it is mapped there. On controller zero, the first few megabytes are
  174. * already mapped in as code at MEM_SV_INTRPT, so in principle we could
  175. * start our data mappings higher up, but for now we don't bother, to avoid
  176. * additional confusion.
  177. *
  178. * One question is whether, on systems with more than 768 Mb and
  179. * controllers of different sizes, to map in a proportionate amount of
  180. * each one, or to try to map the same amount from each controller.
  181. * (E.g. if we have three controllers with 256MB, 1GB, and 256MB
  182. * respectively, do we map 256MB from each, or do we map 128 MB, 512
  183. * MB, and 128 MB respectively?) For now we use a proportionate
  184. * solution like the latter.
  185. *
  186. * The VA/PA mapping demands that we align our decisions at 16 MB
  187. * boundaries so that we can rapidly convert VA to PA.
  188. */
  189. static void *__init setup_pa_va_mapping(void)
  190. {
  191. unsigned long curr_pages = 0;
  192. unsigned long vaddr = PAGE_OFFSET;
  193. nodemask_t highonlynodes = isolnodes;
  194. int i, j;
  195. memset(pbase_map, -1, sizeof(pbase_map));
  196. memset(vbase_map, -1, sizeof(vbase_map));
  197. /* Node zero cannot be isolated for LOWMEM purposes. */
  198. node_clear(0, highonlynodes);
  199. /* Count up the number of pages on non-highonlynodes controllers. */
  200. mappable_physpages = 0;
  201. for_each_online_node(i) {
  202. if (!node_isset(i, highonlynodes))
  203. mappable_physpages +=
  204. node_end_pfn[i] - node_start_pfn[i];
  205. }
  206. for_each_online_node(i) {
  207. unsigned long start = node_start_pfn[i];
  208. unsigned long end = node_end_pfn[i];
  209. unsigned long size = end - start;
  210. unsigned long vaddr_end;
  211. if (node_isset(i, highonlynodes)) {
  212. /* Mark this controller as having no lowmem. */
  213. node_lowmem_end_pfn[i] = start;
  214. continue;
  215. }
  216. curr_pages += size;
  217. if (mappable_physpages > MAXMEM_PFN) {
  218. vaddr_end = PAGE_OFFSET +
  219. (((u64)curr_pages * MAXMEM_PFN /
  220. mappable_physpages)
  221. << PAGE_SHIFT);
  222. } else {
  223. vaddr_end = PAGE_OFFSET + (curr_pages << PAGE_SHIFT);
  224. }
  225. for (j = 0; vaddr < vaddr_end; vaddr += HPAGE_SIZE, ++j) {
  226. unsigned long this_pfn =
  227. start + (j << HUGETLB_PAGE_ORDER);
  228. pbase_map[vaddr >> HPAGE_SHIFT] = this_pfn;
  229. if (vbase_map[__pfn_to_highbits(this_pfn)] ==
  230. (void *)-1)
  231. vbase_map[__pfn_to_highbits(this_pfn)] =
  232. (void *)(vaddr & HPAGE_MASK);
  233. }
  234. node_lowmem_end_pfn[i] = start + (j << HUGETLB_PAGE_ORDER);
  235. BUG_ON(node_lowmem_end_pfn[i] > end);
  236. }
  237. /* Return highest address of any mapped memory. */
  238. return (void *)vaddr;
  239. }
  240. #endif /* CONFIG_HIGHMEM */
  241. /*
  242. * Register our most important memory mappings with the debug stub.
  243. *
  244. * This is up to 4 mappings for lowmem, one mapping per memory
  245. * controller, plus one for our text segment.
  246. */
  247. static void __cpuinit store_permanent_mappings(void)
  248. {
  249. int i;
  250. for_each_online_node(i) {
  251. HV_PhysAddr pa = ((HV_PhysAddr)node_start_pfn[i]) << PAGE_SHIFT;
  252. #ifdef CONFIG_HIGHMEM
  253. HV_PhysAddr high_mapped_pa = node_lowmem_end_pfn[i];
  254. #else
  255. HV_PhysAddr high_mapped_pa = node_end_pfn[i];
  256. #endif
  257. unsigned long pages = high_mapped_pa - node_start_pfn[i];
  258. HV_VirtAddr addr = (HV_VirtAddr) __va(pa);
  259. hv_store_mapping(addr, pages << PAGE_SHIFT, pa);
  260. }
  261. hv_store_mapping((HV_VirtAddr)_stext,
  262. (uint32_t)(_einittext - _stext), 0);
  263. }
  264. /*
  265. * Use hv_inquire_physical() to populate node_{start,end}_pfn[]
  266. * and node_online_map, doing suitable sanity-checking.
  267. * Also set min_low_pfn, max_low_pfn, and max_pfn.
  268. */
  269. static void __init setup_memory(void)
  270. {
  271. int i, j;
  272. int highbits_seen[NR_PA_HIGHBIT_VALUES] = { 0 };
  273. #ifdef CONFIG_HIGHMEM
  274. long highmem_pages;
  275. #endif
  276. #ifndef __tilegx__
  277. int cap;
  278. #endif
  279. #if defined(CONFIG_HIGHMEM) || defined(__tilegx__)
  280. long lowmem_pages;
  281. #endif
  282. /* We are using a char to hold the cpu_2_node[] mapping */
  283. BUILD_BUG_ON(MAX_NUMNODES > 127);
  284. /* Discover the ranges of memory available to us */
  285. for (i = 0; ; ++i) {
  286. unsigned long start, size, end, highbits;
  287. HV_PhysAddrRange range = hv_inquire_physical(i);
  288. if (range.size == 0)
  289. break;
  290. #ifdef CONFIG_FLATMEM
  291. if (i > 0) {
  292. pr_err("Can't use discontiguous PAs: %#llx..%#llx\n",
  293. range.size, range.start + range.size);
  294. continue;
  295. }
  296. #endif
  297. #ifndef __tilegx__
  298. if ((unsigned long)range.start) {
  299. pr_err("Range not at 4GB multiple: %#llx..%#llx\n",
  300. range.start, range.start + range.size);
  301. continue;
  302. }
  303. #endif
  304. if ((range.start & (HPAGE_SIZE-1)) != 0 ||
  305. (range.size & (HPAGE_SIZE-1)) != 0) {
  306. unsigned long long start_pa = range.start;
  307. unsigned long long orig_size = range.size;
  308. range.start = (start_pa + HPAGE_SIZE - 1) & HPAGE_MASK;
  309. range.size -= (range.start - start_pa);
  310. range.size &= HPAGE_MASK;
  311. pr_err("Range not hugepage-aligned: %#llx..%#llx:"
  312. " now %#llx-%#llx\n",
  313. start_pa, start_pa + orig_size,
  314. range.start, range.start + range.size);
  315. }
  316. highbits = __pa_to_highbits(range.start);
  317. if (highbits >= NR_PA_HIGHBIT_VALUES) {
  318. pr_err("PA high bits too high: %#llx..%#llx\n",
  319. range.start, range.start + range.size);
  320. continue;
  321. }
  322. if (highbits_seen[highbits]) {
  323. pr_err("Range overlaps in high bits: %#llx..%#llx\n",
  324. range.start, range.start + range.size);
  325. continue;
  326. }
  327. highbits_seen[highbits] = 1;
  328. if (PFN_DOWN(range.size) > maxnodemem_pfn[i]) {
  329. int max_size = maxnodemem_pfn[i];
  330. if (max_size > 0) {
  331. pr_err("Maxnodemem reduced node %d to"
  332. " %d pages\n", i, max_size);
  333. range.size = PFN_PHYS(max_size);
  334. } else {
  335. pr_err("Maxnodemem disabled node %d\n", i);
  336. continue;
  337. }
  338. }
  339. if (num_physpages + PFN_DOWN(range.size) > maxmem_pfn) {
  340. int max_size = maxmem_pfn - num_physpages;
  341. if (max_size > 0) {
  342. pr_err("Maxmem reduced node %d to %d pages\n",
  343. i, max_size);
  344. range.size = PFN_PHYS(max_size);
  345. } else {
  346. pr_err("Maxmem disabled node %d\n", i);
  347. continue;
  348. }
  349. }
  350. if (i >= MAX_NUMNODES) {
  351. pr_err("Too many PA nodes (#%d): %#llx...%#llx\n",
  352. i, range.size, range.size + range.start);
  353. continue;
  354. }
  355. start = range.start >> PAGE_SHIFT;
  356. size = range.size >> PAGE_SHIFT;
  357. end = start + size;
  358. #ifndef __tilegx__
  359. if (((HV_PhysAddr)end << PAGE_SHIFT) !=
  360. (range.start + range.size)) {
  361. pr_err("PAs too high to represent: %#llx..%#llx\n",
  362. range.start, range.start + range.size);
  363. continue;
  364. }
  365. #endif
  366. #ifdef CONFIG_PCI
  367. /*
  368. * Blocks that overlap the pci reserved region must
  369. * have enough space to hold the maximum percpu data
  370. * region at the top of the range. If there isn't
  371. * enough space above the reserved region, just
  372. * truncate the node.
  373. */
  374. if (start <= pci_reserve_start_pfn &&
  375. end > pci_reserve_start_pfn) {
  376. unsigned int per_cpu_size =
  377. __per_cpu_end - __per_cpu_start;
  378. unsigned int percpu_pages =
  379. NR_CPUS * (PFN_UP(per_cpu_size) >> PAGE_SHIFT);
  380. if (end < pci_reserve_end_pfn + percpu_pages) {
  381. end = pci_reserve_start_pfn;
  382. pr_err("PCI mapping region reduced node %d to"
  383. " %ld pages\n", i, end - start);
  384. }
  385. }
  386. #endif
  387. for (j = __pfn_to_highbits(start);
  388. j <= __pfn_to_highbits(end - 1); j++)
  389. highbits_to_node[j] = i;
  390. node_start_pfn[i] = start;
  391. node_end_pfn[i] = end;
  392. node_controller[i] = range.controller;
  393. num_physpages += size;
  394. max_pfn = end;
  395. /* Mark node as online */
  396. node_set(i, node_online_map);
  397. node_set(i, node_possible_map);
  398. }
  399. #ifndef __tilegx__
  400. /*
  401. * For 4KB pages, mem_map "struct page" data is 1% of the size
  402. * of the physical memory, so can be quite big (640 MB for
  403. * four 16G zones). These structures must be mapped in
  404. * lowmem, and since we currently cap out at about 768 MB,
  405. * it's impractical to try to use this much address space.
  406. * For now, arbitrarily cap the amount of physical memory
  407. * we're willing to use at 8 million pages (32GB of 4KB pages).
  408. */
  409. cap = 8 * 1024 * 1024; /* 8 million pages */
  410. if (num_physpages > cap) {
  411. int num_nodes = num_online_nodes();
  412. int cap_each = cap / num_nodes;
  413. unsigned long dropped_pages = 0;
  414. for (i = 0; i < num_nodes; ++i) {
  415. int size = node_end_pfn[i] - node_start_pfn[i];
  416. if (size > cap_each) {
  417. dropped_pages += (size - cap_each);
  418. node_end_pfn[i] = node_start_pfn[i] + cap_each;
  419. }
  420. }
  421. num_physpages -= dropped_pages;
  422. pr_warning("Only using %ldMB memory;"
  423. " ignoring %ldMB.\n",
  424. num_physpages >> (20 - PAGE_SHIFT),
  425. dropped_pages >> (20 - PAGE_SHIFT));
  426. pr_warning("Consider using a larger page size.\n");
  427. }
  428. #endif
  429. /* Heap starts just above the last loaded address. */
  430. min_low_pfn = PFN_UP((unsigned long)_end - PAGE_OFFSET);
  431. #ifdef CONFIG_HIGHMEM
  432. /* Find where we map lowmem from each controller. */
  433. high_memory = setup_pa_va_mapping();
  434. /* Set max_low_pfn based on what node 0 can directly address. */
  435. max_low_pfn = node_lowmem_end_pfn[0];
  436. lowmem_pages = (mappable_physpages > MAXMEM_PFN) ?
  437. MAXMEM_PFN : mappable_physpages;
  438. highmem_pages = (long) (num_physpages - lowmem_pages);
  439. pr_notice("%ldMB HIGHMEM available.\n",
  440. pages_to_mb(highmem_pages > 0 ? highmem_pages : 0));
  441. pr_notice("%ldMB LOWMEM available.\n",
  442. pages_to_mb(lowmem_pages));
  443. #else
  444. /* Set max_low_pfn based on what node 0 can directly address. */
  445. max_low_pfn = node_end_pfn[0];
  446. #ifndef __tilegx__
  447. if (node_end_pfn[0] > MAXMEM_PFN) {
  448. pr_warning("Only using %ldMB LOWMEM.\n",
  449. MAXMEM>>20);
  450. pr_warning("Use a HIGHMEM enabled kernel.\n");
  451. max_low_pfn = MAXMEM_PFN;
  452. max_pfn = MAXMEM_PFN;
  453. num_physpages = MAXMEM_PFN;
  454. node_end_pfn[0] = MAXMEM_PFN;
  455. } else {
  456. pr_notice("%ldMB memory available.\n",
  457. pages_to_mb(node_end_pfn[0]));
  458. }
  459. for (i = 1; i < MAX_NUMNODES; ++i) {
  460. node_start_pfn[i] = 0;
  461. node_end_pfn[i] = 0;
  462. }
  463. high_memory = __va(node_end_pfn[0]);
  464. #else
  465. lowmem_pages = 0;
  466. for (i = 0; i < MAX_NUMNODES; ++i) {
  467. int pages = node_end_pfn[i] - node_start_pfn[i];
  468. lowmem_pages += pages;
  469. if (pages)
  470. high_memory = pfn_to_kaddr(node_end_pfn[i]);
  471. }
  472. pr_notice("%ldMB memory available.\n",
  473. pages_to_mb(lowmem_pages));
  474. #endif
  475. #endif
  476. }
  477. /*
  478. * On 32-bit machines, we only put bootmem on the low controller,
  479. * since PAs > 4GB can't be used in bootmem. In principle one could
  480. * imagine, e.g., multiple 1 GB controllers all of which could support
  481. * bootmem, but in practice using controllers this small isn't a
  482. * particularly interesting scenario, so we just keep it simple and
  483. * use only the first controller for bootmem on 32-bit machines.
  484. */
  485. static inline int node_has_bootmem(int nid)
  486. {
  487. #ifdef CONFIG_64BIT
  488. return 1;
  489. #else
  490. return nid == 0;
  491. #endif
  492. }
  493. static inline unsigned long alloc_bootmem_pfn(int nid,
  494. unsigned long size,
  495. unsigned long goal)
  496. {
  497. void *kva = __alloc_bootmem_node(NODE_DATA(nid), size,
  498. PAGE_SIZE, goal);
  499. unsigned long pfn = kaddr_to_pfn(kva);
  500. BUG_ON(goal && PFN_PHYS(pfn) != goal);
  501. return pfn;
  502. }
  503. static void __init setup_bootmem_allocator_node(int i)
  504. {
  505. unsigned long start, end, mapsize, mapstart;
  506. if (node_has_bootmem(i)) {
  507. NODE_DATA(i)->bdata = &bootmem_node_data[i];
  508. } else {
  509. /* Share controller zero's bdata for now. */
  510. NODE_DATA(i)->bdata = &bootmem_node_data[0];
  511. return;
  512. }
  513. /* Skip up to after the bss in node 0. */
  514. start = (i == 0) ? min_low_pfn : node_start_pfn[i];
  515. /* Only lowmem, if we're a HIGHMEM build. */
  516. #ifdef CONFIG_HIGHMEM
  517. end = node_lowmem_end_pfn[i];
  518. #else
  519. end = node_end_pfn[i];
  520. #endif
  521. /* No memory here. */
  522. if (end == start)
  523. return;
  524. /* Figure out where the bootmem bitmap is located. */
  525. mapsize = bootmem_bootmap_pages(end - start);
  526. if (i == 0) {
  527. /* Use some space right before the heap on node 0. */
  528. mapstart = start;
  529. start += mapsize;
  530. } else {
  531. /* Allocate bitmap on node 0 to avoid page table issues. */
  532. mapstart = alloc_bootmem_pfn(0, PFN_PHYS(mapsize), 0);
  533. }
  534. /* Initialize a node. */
  535. init_bootmem_node(NODE_DATA(i), mapstart, start, end);
  536. /* Free all the space back into the allocator. */
  537. free_bootmem(PFN_PHYS(start), PFN_PHYS(end - start));
  538. #if defined(CONFIG_PCI)
  539. /*
  540. * Throw away any memory aliased by the PCI region. FIXME: this
  541. * is a temporary hack to work around bug 10502, and needs to be
  542. * fixed properly.
  543. */
  544. if (pci_reserve_start_pfn < end && pci_reserve_end_pfn > start)
  545. reserve_bootmem(PFN_PHYS(pci_reserve_start_pfn),
  546. PFN_PHYS(pci_reserve_end_pfn -
  547. pci_reserve_start_pfn),
  548. BOOTMEM_EXCLUSIVE);
  549. #endif
  550. }
  551. static void __init setup_bootmem_allocator(void)
  552. {
  553. int i;
  554. for (i = 0; i < MAX_NUMNODES; ++i)
  555. setup_bootmem_allocator_node(i);
  556. #ifdef CONFIG_KEXEC
  557. if (crashk_res.start != crashk_res.end)
  558. reserve_bootmem(crashk_res.start, resource_size(&crashk_res), 0);
  559. #endif
  560. }
  561. void *__init alloc_remap(int nid, unsigned long size)
  562. {
  563. int pages = node_end_pfn[nid] - node_start_pfn[nid];
  564. void *map = pfn_to_kaddr(node_memmap_pfn[nid]);
  565. BUG_ON(size != pages * sizeof(struct page));
  566. memset(map, 0, size);
  567. return map;
  568. }
  569. static int __init percpu_size(void)
  570. {
  571. int size = __per_cpu_end - __per_cpu_start;
  572. size += PERCPU_MODULE_RESERVE;
  573. size += PERCPU_DYNAMIC_EARLY_SIZE;
  574. if (size < PCPU_MIN_UNIT_SIZE)
  575. size = PCPU_MIN_UNIT_SIZE;
  576. size = roundup(size, PAGE_SIZE);
  577. /* In several places we assume the per-cpu data fits on a huge page. */
  578. BUG_ON(kdata_huge && size > HPAGE_SIZE);
  579. return size;
  580. }
  581. static void __init zone_sizes_init(void)
  582. {
  583. unsigned long zones_size[MAX_NR_ZONES] = { 0 };
  584. int size = percpu_size();
  585. int num_cpus = smp_height * smp_width;
  586. int i;
  587. for (i = 0; i < num_cpus; ++i)
  588. node_percpu[cpu_to_node(i)] += size;
  589. for_each_online_node(i) {
  590. unsigned long start = node_start_pfn[i];
  591. unsigned long end = node_end_pfn[i];
  592. #ifdef CONFIG_HIGHMEM
  593. unsigned long lowmem_end = node_lowmem_end_pfn[i];
  594. #else
  595. unsigned long lowmem_end = end;
  596. #endif
  597. int memmap_size = (end - start) * sizeof(struct page);
  598. node_free_pfn[i] = start;
  599. /*
  600. * Set aside pages for per-cpu data and the mem_map array.
  601. *
  602. * Since the per-cpu data requires special homecaching,
  603. * if we are in kdata_huge mode, we put it at the end of
  604. * the lowmem region. If we're not in kdata_huge mode,
  605. * we take the per-cpu pages from the bottom of the
  606. * controller, since that avoids fragmenting a huge page
  607. * that users might want. We always take the memmap
  608. * from the bottom of the controller, since with
  609. * kdata_huge that lets it be under a huge TLB entry.
  610. *
  611. * If the user has requested isolnodes for a controller,
  612. * though, there'll be no lowmem, so we just alloc_bootmem
  613. * the memmap. There will be no percpu memory either.
  614. */
  615. if (i != 0 && cpu_isset(i, isolnodes)) {
  616. node_memmap_pfn[i] =
  617. alloc_bootmem_pfn(0, memmap_size, 0);
  618. BUG_ON(node_percpu[i] != 0);
  619. } else if (node_has_bootmem(start)) {
  620. unsigned long goal = 0;
  621. node_memmap_pfn[i] =
  622. alloc_bootmem_pfn(i, memmap_size, 0);
  623. if (kdata_huge)
  624. goal = PFN_PHYS(lowmem_end) - node_percpu[i];
  625. if (node_percpu[i])
  626. node_percpu_pfn[i] =
  627. alloc_bootmem_pfn(i, node_percpu[i],
  628. goal);
  629. } else {
  630. /* In non-bootmem zones, just reserve some pages. */
  631. node_memmap_pfn[i] = node_free_pfn[i];
  632. node_free_pfn[i] += PFN_UP(memmap_size);
  633. if (!kdata_huge) {
  634. node_percpu_pfn[i] = node_free_pfn[i];
  635. node_free_pfn[i] += PFN_UP(node_percpu[i]);
  636. } else {
  637. node_percpu_pfn[i] =
  638. lowmem_end - PFN_UP(node_percpu[i]);
  639. }
  640. }
  641. #ifdef CONFIG_HIGHMEM
  642. if (start > lowmem_end) {
  643. zones_size[ZONE_NORMAL] = 0;
  644. zones_size[ZONE_HIGHMEM] = end - start;
  645. } else {
  646. zones_size[ZONE_NORMAL] = lowmem_end - start;
  647. zones_size[ZONE_HIGHMEM] = end - lowmem_end;
  648. }
  649. #else
  650. zones_size[ZONE_NORMAL] = end - start;
  651. #endif
  652. /* Take zone metadata from controller 0 if we're isolnode. */
  653. if (node_isset(i, isolnodes))
  654. NODE_DATA(i)->bdata = &bootmem_node_data[0];
  655. free_area_init_node(i, zones_size, start, NULL);
  656. printk(KERN_DEBUG " Normal zone: %ld per-cpu pages\n",
  657. PFN_UP(node_percpu[i]));
  658. /* Track the type of memory on each node */
  659. if (zones_size[ZONE_NORMAL])
  660. node_set_state(i, N_NORMAL_MEMORY);
  661. #ifdef CONFIG_HIGHMEM
  662. if (end != start)
  663. node_set_state(i, N_HIGH_MEMORY);
  664. #endif
  665. node_set_online(i);
  666. }
  667. }
  668. #ifdef CONFIG_NUMA
  669. /* which logical CPUs are on which nodes */
  670. struct cpumask node_2_cpu_mask[MAX_NUMNODES] __write_once;
  671. EXPORT_SYMBOL(node_2_cpu_mask);
  672. /* which node each logical CPU is on */
  673. char cpu_2_node[NR_CPUS] __write_once __attribute__((aligned(L2_CACHE_BYTES)));
  674. EXPORT_SYMBOL(cpu_2_node);
  675. /* Return cpu_to_node() except for cpus not yet assigned, which return -1 */
  676. static int __init cpu_to_bound_node(int cpu, struct cpumask* unbound_cpus)
  677. {
  678. if (!cpu_possible(cpu) || cpumask_test_cpu(cpu, unbound_cpus))
  679. return -1;
  680. else
  681. return cpu_to_node(cpu);
  682. }
  683. /* Return number of immediately-adjacent tiles sharing the same NUMA node. */
  684. static int __init node_neighbors(int node, int cpu,
  685. struct cpumask *unbound_cpus)
  686. {
  687. int neighbors = 0;
  688. int w = smp_width;
  689. int h = smp_height;
  690. int x = cpu % w;
  691. int y = cpu / w;
  692. if (x > 0 && cpu_to_bound_node(cpu-1, unbound_cpus) == node)
  693. ++neighbors;
  694. if (x < w-1 && cpu_to_bound_node(cpu+1, unbound_cpus) == node)
  695. ++neighbors;
  696. if (y > 0 && cpu_to_bound_node(cpu-w, unbound_cpus) == node)
  697. ++neighbors;
  698. if (y < h-1 && cpu_to_bound_node(cpu+w, unbound_cpus) == node)
  699. ++neighbors;
  700. return neighbors;
  701. }
  702. static void __init setup_numa_mapping(void)
  703. {
  704. int distance[MAX_NUMNODES][NR_CPUS];
  705. HV_Coord coord;
  706. int cpu, node, cpus, i, x, y;
  707. int num_nodes = num_online_nodes();
  708. struct cpumask unbound_cpus;
  709. nodemask_t default_nodes;
  710. cpumask_clear(&unbound_cpus);
  711. /* Get set of nodes we will use for defaults */
  712. nodes_andnot(default_nodes, node_online_map, isolnodes);
  713. if (nodes_empty(default_nodes)) {
  714. BUG_ON(!node_isset(0, node_online_map));
  715. pr_err("Forcing NUMA node zero available as a default node\n");
  716. node_set(0, default_nodes);
  717. }
  718. /* Populate the distance[] array */
  719. memset(distance, -1, sizeof(distance));
  720. cpu = 0;
  721. for (coord.y = 0; coord.y < smp_height; ++coord.y) {
  722. for (coord.x = 0; coord.x < smp_width;
  723. ++coord.x, ++cpu) {
  724. BUG_ON(cpu >= nr_cpu_ids);
  725. if (!cpu_possible(cpu)) {
  726. cpu_2_node[cpu] = -1;
  727. continue;
  728. }
  729. for_each_node_mask(node, default_nodes) {
  730. HV_MemoryControllerInfo info =
  731. hv_inquire_memory_controller(
  732. coord, node_controller[node]);
  733. distance[node][cpu] =
  734. ABS(info.coord.x) + ABS(info.coord.y);
  735. }
  736. cpumask_set_cpu(cpu, &unbound_cpus);
  737. }
  738. }
  739. cpus = cpu;
  740. /*
  741. * Round-robin through the NUMA nodes until all the cpus are
  742. * assigned. We could be more clever here (e.g. create four
  743. * sorted linked lists on the same set of cpu nodes, and pull
  744. * off them in round-robin sequence, removing from all four
  745. * lists each time) but given the relatively small numbers
  746. * involved, O(n^2) seem OK for a one-time cost.
  747. */
  748. node = first_node(default_nodes);
  749. while (!cpumask_empty(&unbound_cpus)) {
  750. int best_cpu = -1;
  751. int best_distance = INT_MAX;
  752. for (cpu = 0; cpu < cpus; ++cpu) {
  753. if (cpumask_test_cpu(cpu, &unbound_cpus)) {
  754. /*
  755. * Compute metric, which is how much
  756. * closer the cpu is to this memory
  757. * controller than the others, shifted
  758. * up, and then the number of
  759. * neighbors already in the node as an
  760. * epsilon adjustment to try to keep
  761. * the nodes compact.
  762. */
  763. int d = distance[node][cpu] * num_nodes;
  764. for_each_node_mask(i, default_nodes) {
  765. if (i != node)
  766. d -= distance[i][cpu];
  767. }
  768. d *= 8; /* allow space for epsilon */
  769. d -= node_neighbors(node, cpu, &unbound_cpus);
  770. if (d < best_distance) {
  771. best_cpu = cpu;
  772. best_distance = d;
  773. }
  774. }
  775. }
  776. BUG_ON(best_cpu < 0);
  777. cpumask_set_cpu(best_cpu, &node_2_cpu_mask[node]);
  778. cpu_2_node[best_cpu] = node;
  779. cpumask_clear_cpu(best_cpu, &unbound_cpus);
  780. node = next_node(node, default_nodes);
  781. if (node == MAX_NUMNODES)
  782. node = first_node(default_nodes);
  783. }
  784. /* Print out node assignments and set defaults for disabled cpus */
  785. cpu = 0;
  786. for (y = 0; y < smp_height; ++y) {
  787. printk(KERN_DEBUG "NUMA cpu-to-node row %d:", y);
  788. for (x = 0; x < smp_width; ++x, ++cpu) {
  789. if (cpu_to_node(cpu) < 0) {
  790. pr_cont(" -");
  791. cpu_2_node[cpu] = first_node(default_nodes);
  792. } else {
  793. pr_cont(" %d", cpu_to_node(cpu));
  794. }
  795. }
  796. pr_cont("\n");
  797. }
  798. }
  799. static struct cpu cpu_devices[NR_CPUS];
  800. static int __init topology_init(void)
  801. {
  802. int i;
  803. for_each_online_node(i)
  804. register_one_node(i);
  805. for (i = 0; i < smp_height * smp_width; ++i)
  806. register_cpu(&cpu_devices[i], i);
  807. return 0;
  808. }
  809. subsys_initcall(topology_init);
  810. #else /* !CONFIG_NUMA */
  811. #define setup_numa_mapping() do { } while (0)
  812. #endif /* CONFIG_NUMA */
  813. /*
  814. * Initialize hugepage support on this cpu. We do this on all cores
  815. * early in boot: before argument parsing for the boot cpu, and after
  816. * argument parsing but before the init functions run on the secondaries.
  817. * So the values we set up here in the hypervisor may be overridden on
  818. * the boot cpu as arguments are parsed.
  819. */
  820. static __cpuinit void init_super_pages(void)
  821. {
  822. #ifdef CONFIG_HUGETLB_SUPER_PAGES
  823. int i;
  824. for (i = 0; i < HUGE_SHIFT_ENTRIES; ++i)
  825. hv_set_pte_super_shift(i, huge_shift[i]);
  826. #endif
  827. }
  828. /**
  829. * setup_cpu() - Do all necessary per-cpu, tile-specific initialization.
  830. * @boot: Is this the boot cpu?
  831. *
  832. * Called from setup_arch() on the boot cpu, or online_secondary().
  833. */
  834. void __cpuinit setup_cpu(int boot)
  835. {
  836. /* The boot cpu sets up its permanent mappings much earlier. */
  837. if (!boot)
  838. store_permanent_mappings();
  839. /* Allow asynchronous TLB interrupts. */
  840. #if CHIP_HAS_TILE_DMA()
  841. arch_local_irq_unmask(INT_DMATLB_MISS);
  842. arch_local_irq_unmask(INT_DMATLB_ACCESS);
  843. #endif
  844. #if CHIP_HAS_SN_PROC()
  845. arch_local_irq_unmask(INT_SNITLB_MISS);
  846. #endif
  847. #ifdef __tilegx__
  848. arch_local_irq_unmask(INT_SINGLE_STEP_K);
  849. #endif
  850. /*
  851. * Allow user access to many generic SPRs, like the cycle
  852. * counter, PASS/FAIL/DONE, INTERRUPT_CRITICAL_SECTION, etc.
  853. */
  854. __insn_mtspr(SPR_MPL_WORLD_ACCESS_SET_0, 1);
  855. #if CHIP_HAS_SN()
  856. /* Static network is not restricted. */
  857. __insn_mtspr(SPR_MPL_SN_ACCESS_SET_0, 1);
  858. #endif
  859. #if CHIP_HAS_SN_PROC()
  860. __insn_mtspr(SPR_MPL_SN_NOTIFY_SET_0, 1);
  861. __insn_mtspr(SPR_MPL_SN_CPL_SET_0, 1);
  862. #endif
  863. /*
  864. * Set the MPL for interrupt control 0 & 1 to the corresponding
  865. * values. This includes access to the SYSTEM_SAVE and EX_CONTEXT
  866. * SPRs, as well as the interrupt mask.
  867. */
  868. __insn_mtspr(SPR_MPL_INTCTRL_0_SET_0, 1);
  869. __insn_mtspr(SPR_MPL_INTCTRL_1_SET_1, 1);
  870. /* Initialize IRQ support for this cpu. */
  871. setup_irq_regs();
  872. #ifdef CONFIG_HARDWALL
  873. /* Reset the network state on this cpu. */
  874. reset_network_state();
  875. #endif
  876. init_super_pages();
  877. }
  878. #ifdef CONFIG_BLK_DEV_INITRD
  879. /*
  880. * Note that the kernel can potentially support other compression
  881. * techniques than gz, though we don't do so by default. If we ever
  882. * decide to do so we can either look for other filename extensions,
  883. * or just allow a file with this name to be compressed with an
  884. * arbitrary compressor (somewhat counterintuitively).
  885. */
  886. static int __initdata set_initramfs_file;
  887. static char __initdata initramfs_file[128] = "initramfs.cpio.gz";
  888. static int __init setup_initramfs_file(char *str)
  889. {
  890. if (str == NULL)
  891. return -EINVAL;
  892. strncpy(initramfs_file, str, sizeof(initramfs_file) - 1);
  893. set_initramfs_file = 1;
  894. return 0;
  895. }
  896. early_param("initramfs_file", setup_initramfs_file);
  897. /*
  898. * We look for an "initramfs.cpio.gz" file in the hvfs.
  899. * If there is one, we allocate some memory for it and it will be
  900. * unpacked to the initramfs.
  901. */
  902. static void __init load_hv_initrd(void)
  903. {
  904. HV_FS_StatInfo stat;
  905. int fd, rc;
  906. void *initrd;
  907. fd = hv_fs_findfile((HV_VirtAddr) initramfs_file);
  908. if (fd == HV_ENOENT) {
  909. if (set_initramfs_file)
  910. pr_warning("No such hvfs initramfs file '%s'\n",
  911. initramfs_file);
  912. return;
  913. }
  914. BUG_ON(fd < 0);
  915. stat = hv_fs_fstat(fd);
  916. BUG_ON(stat.size < 0);
  917. if (stat.flags & HV_FS_ISDIR) {
  918. pr_warning("Ignoring hvfs file '%s': it's a directory.\n",
  919. initramfs_file);
  920. return;
  921. }
  922. initrd = alloc_bootmem_pages(stat.size);
  923. rc = hv_fs_pread(fd, (HV_VirtAddr) initrd, stat.size, 0);
  924. if (rc != stat.size) {
  925. pr_err("Error reading %d bytes from hvfs file '%s': %d\n",
  926. stat.size, initramfs_file, rc);
  927. free_initrd_mem((unsigned long) initrd, stat.size);
  928. return;
  929. }
  930. initrd_start = (unsigned long) initrd;
  931. initrd_end = initrd_start + stat.size;
  932. }
  933. void __init free_initrd_mem(unsigned long begin, unsigned long end)
  934. {
  935. free_bootmem(__pa(begin), end - begin);
  936. }
  937. #else
  938. static inline void load_hv_initrd(void) {}
  939. #endif /* CONFIG_BLK_DEV_INITRD */
  940. static void __init validate_hv(void)
  941. {
  942. /*
  943. * It may already be too late, but let's check our built-in
  944. * configuration against what the hypervisor is providing.
  945. */
  946. unsigned long glue_size = hv_sysconf(HV_SYSCONF_GLUE_SIZE);
  947. int hv_page_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_SMALL);
  948. int hv_hpage_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_LARGE);
  949. HV_ASIDRange asid_range;
  950. #ifndef CONFIG_SMP
  951. HV_Topology topology = hv_inquire_topology();
  952. BUG_ON(topology.coord.x != 0 || topology.coord.y != 0);
  953. if (topology.width != 1 || topology.height != 1) {
  954. pr_warning("Warning: booting UP kernel on %dx%d grid;"
  955. " will ignore all but first tile.\n",
  956. topology.width, topology.height);
  957. }
  958. #endif
  959. if (PAGE_OFFSET + HV_GLUE_START_CPA + glue_size > (unsigned long)_text)
  960. early_panic("Hypervisor glue size %ld is too big!\n",
  961. glue_size);
  962. if (hv_page_size != PAGE_SIZE)
  963. early_panic("Hypervisor page size %#x != our %#lx\n",
  964. hv_page_size, PAGE_SIZE);
  965. if (hv_hpage_size != HPAGE_SIZE)
  966. early_panic("Hypervisor huge page size %#x != our %#lx\n",
  967. hv_hpage_size, HPAGE_SIZE);
  968. #ifdef CONFIG_SMP
  969. /*
  970. * Some hypervisor APIs take a pointer to a bitmap array
  971. * whose size is at least the number of cpus on the chip.
  972. * We use a struct cpumask for this, so it must be big enough.
  973. */
  974. if ((smp_height * smp_width) > nr_cpu_ids)
  975. early_panic("Hypervisor %d x %d grid too big for Linux"
  976. " NR_CPUS %d\n", smp_height, smp_width,
  977. nr_cpu_ids);
  978. #endif
  979. /*
  980. * Check that we're using allowed ASIDs, and initialize the
  981. * various asid variables to their appropriate initial states.
  982. */
  983. asid_range = hv_inquire_asid(0);
  984. __get_cpu_var(current_asid) = min_asid = asid_range.start;
  985. max_asid = asid_range.start + asid_range.size - 1;
  986. if (hv_confstr(HV_CONFSTR_CHIP_MODEL, (HV_VirtAddr)chip_model,
  987. sizeof(chip_model)) < 0) {
  988. pr_err("Warning: HV_CONFSTR_CHIP_MODEL not available\n");
  989. strlcpy(chip_model, "unknown", sizeof(chip_model));
  990. }
  991. }
  992. static void __init validate_va(void)
  993. {
  994. #ifndef __tilegx__ /* FIXME: GX: probably some validation relevant here */
  995. /*
  996. * Similarly, make sure we're only using allowed VAs.
  997. * We assume we can contiguously use MEM_USER_INTRPT .. MEM_HV_INTRPT,
  998. * and 0 .. KERNEL_HIGH_VADDR.
  999. * In addition, make sure we CAN'T use the end of memory, since
  1000. * we use the last chunk of each pgd for the pgd_list.
  1001. */
  1002. int i, user_kernel_ok = 0;
  1003. unsigned long max_va = 0;
  1004. unsigned long list_va =
  1005. ((PGD_LIST_OFFSET / sizeof(pgd_t)) << PGDIR_SHIFT);
  1006. for (i = 0; ; ++i) {
  1007. HV_VirtAddrRange range = hv_inquire_virtual(i);
  1008. if (range.size == 0)
  1009. break;
  1010. if (range.start <= MEM_USER_INTRPT &&
  1011. range.start + range.size >= MEM_HV_INTRPT)
  1012. user_kernel_ok = 1;
  1013. if (range.start == 0)
  1014. max_va = range.size;
  1015. BUG_ON(range.start + range.size > list_va);
  1016. }
  1017. if (!user_kernel_ok)
  1018. early_panic("Hypervisor not configured for user/kernel VAs\n");
  1019. if (max_va == 0)
  1020. early_panic("Hypervisor not configured for low VAs\n");
  1021. if (max_va < KERNEL_HIGH_VADDR)
  1022. early_panic("Hypervisor max VA %#lx smaller than %#lx\n",
  1023. max_va, KERNEL_HIGH_VADDR);
  1024. /* Kernel PCs must have their high bit set; see intvec.S. */
  1025. if ((long)VMALLOC_START >= 0)
  1026. early_panic(
  1027. "Linux VMALLOC region below the 2GB line (%#lx)!\n"
  1028. "Reconfigure the kernel with fewer NR_HUGE_VMAPS\n"
  1029. "or smaller VMALLOC_RESERVE.\n",
  1030. VMALLOC_START);
  1031. #endif
  1032. }
  1033. /*
  1034. * cpu_lotar_map lists all the cpus that are valid for the supervisor
  1035. * to cache data on at a page level, i.e. what cpus can be placed in
  1036. * the LOTAR field of a PTE. It is equivalent to the set of possible
  1037. * cpus plus any other cpus that are willing to share their cache.
  1038. * It is set by hv_inquire_tiles(HV_INQ_TILES_LOTAR).
  1039. */
  1040. struct cpumask __write_once cpu_lotar_map;
  1041. EXPORT_SYMBOL(cpu_lotar_map);
  1042. #if CHIP_HAS_CBOX_HOME_MAP()
  1043. /*
  1044. * hash_for_home_map lists all the tiles that hash-for-home data
  1045. * will be cached on. Note that this may includes tiles that are not
  1046. * valid for this supervisor to use otherwise (e.g. if a hypervisor
  1047. * device is being shared between multiple supervisors).
  1048. * It is set by hv_inquire_tiles(HV_INQ_TILES_HFH_CACHE).
  1049. */
  1050. struct cpumask hash_for_home_map;
  1051. EXPORT_SYMBOL(hash_for_home_map);
  1052. #endif
  1053. /*
  1054. * cpu_cacheable_map lists all the cpus whose caches the hypervisor can
  1055. * flush on our behalf. It is set to cpu_possible_mask OR'ed with
  1056. * hash_for_home_map, and it is what should be passed to
  1057. * hv_flush_remote() to flush all caches. Note that if there are
  1058. * dedicated hypervisor driver tiles that have authorized use of their
  1059. * cache, those tiles will only appear in cpu_lotar_map, NOT in
  1060. * cpu_cacheable_map, as they are a special case.
  1061. */
  1062. struct cpumask __write_once cpu_cacheable_map;
  1063. EXPORT_SYMBOL(cpu_cacheable_map);
  1064. static __initdata struct cpumask disabled_map;
  1065. static int __init disabled_cpus(char *str)
  1066. {
  1067. int boot_cpu = smp_processor_id();
  1068. if (str == NULL || cpulist_parse_crop(str, &disabled_map) != 0)
  1069. return -EINVAL;
  1070. if (cpumask_test_cpu(boot_cpu, &disabled_map)) {
  1071. pr_err("disabled_cpus: can't disable boot cpu %d\n", boot_cpu);
  1072. cpumask_clear_cpu(boot_cpu, &disabled_map);
  1073. }
  1074. return 0;
  1075. }
  1076. early_param("disabled_cpus", disabled_cpus);
  1077. void __init print_disabled_cpus(void)
  1078. {
  1079. if (!cpumask_empty(&disabled_map)) {
  1080. char buf[100];
  1081. cpulist_scnprintf(buf, sizeof(buf), &disabled_map);
  1082. pr_info("CPUs not available for Linux: %s\n", buf);
  1083. }
  1084. }
  1085. static void __init setup_cpu_maps(void)
  1086. {
  1087. struct cpumask hv_disabled_map, cpu_possible_init;
  1088. int boot_cpu = smp_processor_id();
  1089. int cpus, i, rc;
  1090. /* Learn which cpus are allowed by the hypervisor. */
  1091. rc = hv_inquire_tiles(HV_INQ_TILES_AVAIL,
  1092. (HV_VirtAddr) cpumask_bits(&cpu_possible_init),
  1093. sizeof(cpu_cacheable_map));
  1094. if (rc < 0)
  1095. early_panic("hv_inquire_tiles(AVAIL) failed: rc %d\n", rc);
  1096. if (!cpumask_test_cpu(boot_cpu, &cpu_possible_init))
  1097. early_panic("Boot CPU %d disabled by hypervisor!\n", boot_cpu);
  1098. /* Compute the cpus disabled by the hvconfig file. */
  1099. cpumask_complement(&hv_disabled_map, &cpu_possible_init);
  1100. /* Include them with the cpus disabled by "disabled_cpus". */
  1101. cpumask_or(&disabled_map, &disabled_map, &hv_disabled_map);
  1102. /*
  1103. * Disable every cpu after "setup_max_cpus". But don't mark
  1104. * as disabled the cpus that are outside of our initial rectangle,
  1105. * since that turns out to be confusing.
  1106. */
  1107. cpus = 1; /* this cpu */
  1108. cpumask_set_cpu(boot_cpu, &disabled_map); /* ignore this cpu */
  1109. for (i = 0; cpus < setup_max_cpus; ++i)
  1110. if (!cpumask_test_cpu(i, &disabled_map))
  1111. ++cpus;
  1112. for (; i < smp_height * smp_width; ++i)
  1113. cpumask_set_cpu(i, &disabled_map);
  1114. cpumask_clear_cpu(boot_cpu, &disabled_map); /* reset this cpu */
  1115. for (i = smp_height * smp_width; i < NR_CPUS; ++i)
  1116. cpumask_clear_cpu(i, &disabled_map);
  1117. /*
  1118. * Setup cpu_possible map as every cpu allocated to us, minus
  1119. * the results of any "disabled_cpus" settings.
  1120. */
  1121. cpumask_andnot(&cpu_possible_init, &cpu_possible_init, &disabled_map);
  1122. init_cpu_possible(&cpu_possible_init);
  1123. /* Learn which cpus are valid for LOTAR caching. */
  1124. rc = hv_inquire_tiles(HV_INQ_TILES_LOTAR,
  1125. (HV_VirtAddr) cpumask_bits(&cpu_lotar_map),
  1126. sizeof(cpu_lotar_map));
  1127. if (rc < 0) {
  1128. pr_err("warning: no HV_INQ_TILES_LOTAR; using AVAIL\n");
  1129. cpu_lotar_map = *cpu_possible_mask;
  1130. }
  1131. #if CHIP_HAS_CBOX_HOME_MAP()
  1132. /* Retrieve set of CPUs used for hash-for-home caching */
  1133. rc = hv_inquire_tiles(HV_INQ_TILES_HFH_CACHE,
  1134. (HV_VirtAddr) hash_for_home_map.bits,
  1135. sizeof(hash_for_home_map));
  1136. if (rc < 0)
  1137. early_panic("hv_inquire_tiles(HFH_CACHE) failed: rc %d\n", rc);
  1138. cpumask_or(&cpu_cacheable_map, cpu_possible_mask, &hash_for_home_map);
  1139. #else
  1140. cpu_cacheable_map = *cpu_possible_mask;
  1141. #endif
  1142. }
  1143. static int __init dataplane(char *str)
  1144. {
  1145. pr_warning("WARNING: dataplane support disabled in this kernel\n");
  1146. return 0;
  1147. }
  1148. early_param("dataplane", dataplane);
  1149. #ifdef CONFIG_CMDLINE_BOOL
  1150. static char __initdata builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE;
  1151. #endif
  1152. void __init setup_arch(char **cmdline_p)
  1153. {
  1154. int len;
  1155. #if defined(CONFIG_CMDLINE_BOOL) && defined(CONFIG_CMDLINE_OVERRIDE)
  1156. len = hv_get_command_line((HV_VirtAddr) boot_command_line,
  1157. COMMAND_LINE_SIZE);
  1158. if (boot_command_line[0])
  1159. pr_warning("WARNING: ignoring dynamic command line \"%s\"\n",
  1160. boot_command_line);
  1161. strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
  1162. #else
  1163. char *hv_cmdline;
  1164. #if defined(CONFIG_CMDLINE_BOOL)
  1165. if (builtin_cmdline[0]) {
  1166. int builtin_len = strlcpy(boot_command_line, builtin_cmdline,
  1167. COMMAND_LINE_SIZE);
  1168. if (builtin_len < COMMAND_LINE_SIZE-1)
  1169. boot_command_line[builtin_len++] = ' ';
  1170. hv_cmdline = &boot_command_line[builtin_len];
  1171. len = COMMAND_LINE_SIZE - builtin_len;
  1172. } else
  1173. #endif
  1174. {
  1175. hv_cmdline = boot_command_line;
  1176. len = COMMAND_LINE_SIZE;
  1177. }
  1178. len = hv_get_command_line((HV_VirtAddr) hv_cmdline, len);
  1179. if (len < 0 || len > COMMAND_LINE_SIZE)
  1180. early_panic("hv_get_command_line failed: %d\n", len);
  1181. #endif
  1182. *cmdline_p = boot_command_line;
  1183. /* Set disabled_map and setup_max_cpus very early */
  1184. parse_early_param();
  1185. /* Make sure the kernel is compatible with the hypervisor. */
  1186. validate_hv();
  1187. validate_va();
  1188. setup_cpu_maps();
  1189. #ifdef CONFIG_PCI
  1190. /*
  1191. * Initialize the PCI structures. This is done before memory
  1192. * setup so that we know whether or not a pci_reserve region
  1193. * is necessary.
  1194. */
  1195. if (tile_pci_init() == 0)
  1196. pci_reserve_mb = 0;
  1197. /* PCI systems reserve a region just below 4GB for mapping iomem. */
  1198. pci_reserve_end_pfn = (1 << (32 - PAGE_SHIFT));
  1199. pci_reserve_start_pfn = pci_reserve_end_pfn -
  1200. (pci_reserve_mb << (20 - PAGE_SHIFT));
  1201. #endif
  1202. init_mm.start_code = (unsigned long) _text;
  1203. init_mm.end_code = (unsigned long) _etext;
  1204. init_mm.end_data = (unsigned long) _edata;
  1205. init_mm.brk = (unsigned long) _end;
  1206. setup_memory();
  1207. store_permanent_mappings();
  1208. setup_bootmem_allocator();
  1209. /*
  1210. * NOTE: before this point _nobody_ is allowed to allocate
  1211. * any memory using the bootmem allocator.
  1212. */
  1213. paging_init();
  1214. setup_numa_mapping();
  1215. zone_sizes_init();
  1216. set_page_homes();
  1217. setup_cpu(1);
  1218. setup_clock();
  1219. load_hv_initrd();
  1220. }
  1221. /*
  1222. * Set up per-cpu memory.
  1223. */
  1224. unsigned long __per_cpu_offset[NR_CPUS] __write_once;
  1225. EXPORT_SYMBOL(__per_cpu_offset);
  1226. static size_t __initdata pfn_offset[MAX_NUMNODES] = { 0 };
  1227. static unsigned long __initdata percpu_pfn[NR_CPUS] = { 0 };
  1228. /*
  1229. * As the percpu code allocates pages, we return the pages from the
  1230. * end of the node for the specified cpu.
  1231. */
  1232. static void *__init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)
  1233. {
  1234. int nid = cpu_to_node(cpu);
  1235. unsigned long pfn = node_percpu_pfn[nid] + pfn_offset[nid];
  1236. BUG_ON(size % PAGE_SIZE != 0);
  1237. pfn_offset[nid] += size / PAGE_SIZE;
  1238. BUG_ON(node_percpu[nid] < size);
  1239. node_percpu[nid] -= size;
  1240. if (percpu_pfn[cpu] == 0)
  1241. percpu_pfn[cpu] = pfn;
  1242. return pfn_to_kaddr(pfn);
  1243. }
  1244. /*
  1245. * Pages reserved for percpu memory are not freeable, and in any case we are
  1246. * on a short path to panic() in setup_per_cpu_area() at this point anyway.
  1247. */
  1248. static void __init pcpu_fc_free(void *ptr, size_t size)
  1249. {
  1250. }
  1251. /*
  1252. * Set up vmalloc page tables using bootmem for the percpu code.
  1253. */
  1254. static void __init pcpu_fc_populate_pte(unsigned long addr)
  1255. {
  1256. pgd_t *pgd;
  1257. pud_t *pud;
  1258. pmd_t *pmd;
  1259. pte_t *pte;
  1260. BUG_ON(pgd_addr_invalid(addr));
  1261. if (addr < VMALLOC_START || addr >= VMALLOC_END)
  1262. panic("PCPU addr %#lx outside vmalloc range %#lx..%#lx;"
  1263. " try increasing CONFIG_VMALLOC_RESERVE\n",
  1264. addr, VMALLOC_START, VMALLOC_END);
  1265. pgd = swapper_pg_dir + pgd_index(addr);
  1266. pud = pud_offset(pgd, addr);
  1267. BUG_ON(!pud_present(*pud));
  1268. pmd = pmd_offset(pud, addr);
  1269. if (pmd_present(*pmd)) {
  1270. BUG_ON(pmd_huge_page(*pmd));
  1271. } else {
  1272. pte = __alloc_bootmem(L2_KERNEL_PGTABLE_SIZE,
  1273. HV_PAGE_TABLE_ALIGN, 0);
  1274. pmd_populate_kernel(&init_mm, pmd, pte);
  1275. }
  1276. }
  1277. void __init setup_per_cpu_areas(void)
  1278. {
  1279. struct page *pg;
  1280. unsigned long delta, pfn, lowmem_va;
  1281. unsigned long size = percpu_size();
  1282. char *ptr;
  1283. int rc, cpu, i;
  1284. rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_fc_alloc,
  1285. pcpu_fc_free, pcpu_fc_populate_pte);
  1286. if (rc < 0)
  1287. panic("Cannot initialize percpu area (err=%d)", rc);
  1288. delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
  1289. for_each_possible_cpu(cpu) {
  1290. __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
  1291. /* finv the copy out of cache so we can change homecache */
  1292. ptr = pcpu_base_addr + pcpu_unit_offsets[cpu];
  1293. __finv_buffer(ptr, size);
  1294. pfn = percpu_pfn[cpu];
  1295. /* Rewrite the page tables to cache on that cpu */
  1296. pg = pfn_to_page(pfn);
  1297. for (i = 0; i < size; i += PAGE_SIZE, ++pfn, ++pg) {
  1298. /* Update the vmalloc mapping and page home. */
  1299. unsigned long addr = (unsigned long)ptr + i;
  1300. pte_t *ptep = virt_to_pte(NULL, addr);
  1301. pte_t pte = *ptep;
  1302. BUG_ON(pfn != pte_pfn(pte));
  1303. pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_TILE_L3);
  1304. pte = set_remote_cache_cpu(pte, cpu);
  1305. set_pte_at(&init_mm, addr, ptep, pte);
  1306. /* Update the lowmem mapping for consistency. */
  1307. lowmem_va = (unsigned long)pfn_to_kaddr(pfn);
  1308. ptep = virt_to_pte(NULL, lowmem_va);
  1309. if (pte_huge(*ptep)) {
  1310. printk(KERN_DEBUG "early shatter of huge page"
  1311. " at %#lx\n", lowmem_va);
  1312. shatter_pmd((pmd_t *)ptep);
  1313. ptep = virt_to_pte(NULL, lowmem_va);
  1314. BUG_ON(pte_huge(*ptep));
  1315. }
  1316. BUG_ON(pfn != pte_pfn(*ptep));
  1317. set_pte_at(&init_mm, lowmem_va, ptep, pte);
  1318. }
  1319. }
  1320. /* Set our thread pointer appropriately. */
  1321. set_my_cpu_offset(__per_cpu_offset[smp_processor_id()]);
  1322. /* Make sure the finv's have completed. */
  1323. mb_incoherent();
  1324. /* Flush the TLB so we reference it properly from here on out. */
  1325. local_flush_tlb_all();
  1326. }
  1327. static struct resource data_resource = {
  1328. .name = "Kernel data",
  1329. .start = 0,
  1330. .end = 0,
  1331. .flags = IORESOURCE_BUSY | IORESOURCE_MEM
  1332. };
  1333. static struct resource code_resource = {
  1334. .name = "Kernel code",
  1335. .start = 0,
  1336. .end = 0,
  1337. .flags = IORESOURCE_BUSY | IORESOURCE_MEM
  1338. };
  1339. /*
  1340. * We reserve all resources above 4GB so that PCI won't try to put
  1341. * mappings above 4GB; the standard allows that for some devices but
  1342. * the probing code trunates values to 32 bits.
  1343. */
  1344. #ifdef CONFIG_PCI
  1345. static struct resource* __init
  1346. insert_non_bus_resource(void)
  1347. {
  1348. struct resource *res =
  1349. kzalloc(sizeof(struct resource), GFP_ATOMIC);
  1350. res->name = "Non-Bus Physical Address Space";
  1351. res->start = (1ULL << 32);
  1352. res->end = -1LL;
  1353. res->flags = IORESOURCE_BUSY | IORESOURCE_MEM;
  1354. if (insert_resource(&iomem_resource, res)) {
  1355. kfree(res);
  1356. return NULL;
  1357. }
  1358. return res;
  1359. }
  1360. #endif
  1361. static struct resource* __init
  1362. insert_ram_resource(u64 start_pfn, u64 end_pfn)
  1363. {
  1364. struct resource *res =
  1365. kzalloc(sizeof(struct resource), GFP_ATOMIC);
  1366. res->name = "System RAM";
  1367. res->start = start_pfn << PAGE_SHIFT;
  1368. res->end = (end_pfn << PAGE_SHIFT) - 1;
  1369. res->flags = IORESOURCE_BUSY | IORESOURCE_MEM;
  1370. if (insert_resource(&iomem_resource, res)) {
  1371. kfree(res);
  1372. return NULL;
  1373. }
  1374. return res;
  1375. }
  1376. /*
  1377. * Request address space for all standard resources
  1378. *
  1379. * If the system includes PCI root complex drivers, we need to create
  1380. * a window just below 4GB where PCI BARs can be mapped.
  1381. */
  1382. static int __init request_standard_resources(void)
  1383. {
  1384. int i;
  1385. enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET };
  1386. iomem_resource.end = -1LL;
  1387. #ifdef CONFIG_PCI
  1388. insert_non_bus_resource();
  1389. #endif
  1390. for_each_online_node(i) {
  1391. u64 start_pfn = node_start_pfn[i];
  1392. u64 end_pfn = node_end_pfn[i];
  1393. #ifdef CONFIG_PCI
  1394. if (start_pfn <= pci_reserve_start_pfn &&
  1395. end_pfn > pci_reserve_start_pfn) {
  1396. if (end_pfn > pci_reserve_end_pfn)
  1397. insert_ram_resource(pci_reserve_end_pfn,
  1398. end_pfn);
  1399. end_pfn = pci_reserve_start_pfn;
  1400. }
  1401. #endif
  1402. insert_ram_resource(start_pfn, end_pfn);
  1403. }
  1404. code_resource.start = __pa(_text - CODE_DELTA);
  1405. code_resource.end = __pa(_etext - CODE_DELTA)-1;
  1406. data_resource.start = __pa(_sdata);
  1407. data_resource.end = __pa(_end)-1;
  1408. insert_resource(&iomem_resource, &code_resource);
  1409. insert_resource(&iomem_resource, &data_resource);
  1410. #ifdef CONFIG_KEXEC
  1411. insert_resource(&iomem_resource, &crashk_res);
  1412. #endif
  1413. return 0;
  1414. }
  1415. subsys_initcall(request_standard_resources);