smp.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098
  1. /*
  2. * arch/s390/kernel/smp.c
  3. *
  4. * Copyright IBM Corp. 1999, 2009
  5. * Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
  6. * Martin Schwidefsky (schwidefsky@de.ibm.com)
  7. * Heiko Carstens (heiko.carstens@de.ibm.com)
  8. *
  9. * based on other smp stuff by
  10. * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
  11. * (c) 1998 Ingo Molnar
  12. *
  13. * We work with logical cpu numbering everywhere we can. The only
  14. * functions using the real cpu address (got from STAP) are the sigp
  15. * functions. For all other functions we use the identity mapping.
  16. * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
  17. * used e.g. to find the idle task belonging to a logical cpu. Every array
  18. * in the kernel is sorted by the logical cpu number and not by the physical
  19. * one which is causing all the confusion with __cpu_logical_map and
  20. * cpu_number_map in other architectures.
  21. */
  22. #define KMSG_COMPONENT "cpu"
  23. #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  24. #include <linux/module.h>
  25. #include <linux/init.h>
  26. #include <linux/mm.h>
  27. #include <linux/err.h>
  28. #include <linux/spinlock.h>
  29. #include <linux/kernel_stat.h>
  30. #include <linux/delay.h>
  31. #include <linux/cache.h>
  32. #include <linux/interrupt.h>
  33. #include <linux/irqflags.h>
  34. #include <linux/cpu.h>
  35. #include <linux/timex.h>
  36. #include <linux/bootmem.h>
  37. #include <asm/asm-offsets.h>
  38. #include <asm/ipl.h>
  39. #include <asm/setup.h>
  40. #include <asm/sigp.h>
  41. #include <asm/pgalloc.h>
  42. #include <asm/irq.h>
  43. #include <asm/s390_ext.h>
  44. #include <asm/cpcmd.h>
  45. #include <asm/tlbflush.h>
  46. #include <asm/timer.h>
  47. #include <asm/lowcore.h>
  48. #include <asm/sclp.h>
  49. #include <asm/cputime.h>
  50. #include <asm/vdso.h>
  51. #include <asm/cpu.h>
  52. #include "entry.h"
  53. /* logical cpu to cpu address */
  54. unsigned short __cpu_logical_map[NR_CPUS];
  55. static struct task_struct *current_set[NR_CPUS];
  56. static u8 smp_cpu_type;
  57. static int smp_use_sigp_detection;
  58. enum s390_cpu_state {
  59. CPU_STATE_STANDBY,
  60. CPU_STATE_CONFIGURED,
  61. };
  62. DEFINE_MUTEX(smp_cpu_state_mutex);
  63. int smp_cpu_polarization[NR_CPUS];
  64. static int smp_cpu_state[NR_CPUS];
  65. static int cpu_management;
  66. static DEFINE_PER_CPU(struct cpu, cpu_devices);
  67. static void smp_ext_bitcall(int, int);
  68. static int raw_cpu_stopped(int cpu)
  69. {
  70. u32 status;
  71. switch (raw_sigp_ps(&status, 0, cpu, sigp_sense)) {
  72. case sigp_status_stored:
  73. /* Check for stopped and check stop state */
  74. if (status & 0x50)
  75. return 1;
  76. break;
  77. default:
  78. break;
  79. }
  80. return 0;
  81. }
  82. static inline int cpu_stopped(int cpu)
  83. {
  84. return raw_cpu_stopped(cpu_logical_map(cpu));
  85. }
  86. void smp_switch_to_ipl_cpu(void (*func)(void *), void *data)
  87. {
  88. struct _lowcore *lc, *current_lc;
  89. struct stack_frame *sf;
  90. struct pt_regs *regs;
  91. unsigned long sp;
  92. if (smp_processor_id() == 0)
  93. func(data);
  94. __load_psw_mask(PSW_BASE_BITS | PSW_DEFAULT_KEY);
  95. /* Disable lowcore protection */
  96. __ctl_clear_bit(0, 28);
  97. current_lc = lowcore_ptr[smp_processor_id()];
  98. lc = lowcore_ptr[0];
  99. if (!lc)
  100. lc = current_lc;
  101. lc->restart_psw.mask = PSW_BASE_BITS | PSW_DEFAULT_KEY;
  102. lc->restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) smp_restart_cpu;
  103. if (!cpu_online(0))
  104. smp_switch_to_cpu(func, data, 0, stap(), __cpu_logical_map[0]);
  105. while (sigp(0, sigp_stop_and_store_status) == sigp_busy)
  106. cpu_relax();
  107. sp = lc->panic_stack;
  108. sp -= sizeof(struct pt_regs);
  109. regs = (struct pt_regs *) sp;
  110. memcpy(&regs->gprs, &current_lc->gpregs_save_area, sizeof(regs->gprs));
  111. regs->psw = lc->psw_save_area;
  112. sp -= STACK_FRAME_OVERHEAD;
  113. sf = (struct stack_frame *) sp;
  114. sf->back_chain = regs->gprs[15];
  115. smp_switch_to_cpu(func, data, sp, stap(), __cpu_logical_map[0]);
  116. }
  117. void smp_send_stop(void)
  118. {
  119. int cpu, rc;
  120. /* Disable all interrupts/machine checks */
  121. __load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
  122. trace_hardirqs_off();
  123. /* stop all processors */
  124. for_each_online_cpu(cpu) {
  125. if (cpu == smp_processor_id())
  126. continue;
  127. do {
  128. rc = sigp(cpu, sigp_stop);
  129. } while (rc == sigp_busy);
  130. while (!cpu_stopped(cpu))
  131. cpu_relax();
  132. }
  133. }
  134. /*
  135. * This is the main routine where commands issued by other
  136. * cpus are handled.
  137. */
  138. static void do_ext_call_interrupt(__u16 code)
  139. {
  140. unsigned long bits;
  141. /*
  142. * handle bit signal external calls
  143. *
  144. * For the ec_schedule signal we have to do nothing. All the work
  145. * is done automatically when we return from the interrupt.
  146. */
  147. bits = xchg(&S390_lowcore.ext_call_fast, 0);
  148. if (test_bit(ec_call_function, &bits))
  149. generic_smp_call_function_interrupt();
  150. if (test_bit(ec_call_function_single, &bits))
  151. generic_smp_call_function_single_interrupt();
  152. }
  153. /*
  154. * Send an external call sigp to another cpu and return without waiting
  155. * for its completion.
  156. */
  157. static void smp_ext_bitcall(int cpu, int sig)
  158. {
  159. /*
  160. * Set signaling bit in lowcore of target cpu and kick it
  161. */
  162. set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
  163. while (sigp(cpu, sigp_emergency_signal) == sigp_busy)
  164. udelay(10);
  165. }
  166. void arch_send_call_function_ipi_mask(const struct cpumask *mask)
  167. {
  168. int cpu;
  169. for_each_cpu(cpu, mask)
  170. smp_ext_bitcall(cpu, ec_call_function);
  171. }
  172. void arch_send_call_function_single_ipi(int cpu)
  173. {
  174. smp_ext_bitcall(cpu, ec_call_function_single);
  175. }
  176. #ifndef CONFIG_64BIT
  177. /*
  178. * this function sends a 'purge tlb' signal to another CPU.
  179. */
  180. static void smp_ptlb_callback(void *info)
  181. {
  182. __tlb_flush_local();
  183. }
  184. void smp_ptlb_all(void)
  185. {
  186. on_each_cpu(smp_ptlb_callback, NULL, 1);
  187. }
  188. EXPORT_SYMBOL(smp_ptlb_all);
  189. #endif /* ! CONFIG_64BIT */
  190. /*
  191. * this function sends a 'reschedule' IPI to another CPU.
  192. * it goes straight through and wastes no time serializing
  193. * anything. Worst case is that we lose a reschedule ...
  194. */
  195. void smp_send_reschedule(int cpu)
  196. {
  197. smp_ext_bitcall(cpu, ec_schedule);
  198. }
  199. /*
  200. * parameter area for the set/clear control bit callbacks
  201. */
  202. struct ec_creg_mask_parms {
  203. unsigned long orvals[16];
  204. unsigned long andvals[16];
  205. };
  206. /*
  207. * callback for setting/clearing control bits
  208. */
  209. static void smp_ctl_bit_callback(void *info)
  210. {
  211. struct ec_creg_mask_parms *pp = info;
  212. unsigned long cregs[16];
  213. int i;
  214. __ctl_store(cregs, 0, 15);
  215. for (i = 0; i <= 15; i++)
  216. cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
  217. __ctl_load(cregs, 0, 15);
  218. }
  219. /*
  220. * Set a bit in a control register of all cpus
  221. */
  222. void smp_ctl_set_bit(int cr, int bit)
  223. {
  224. struct ec_creg_mask_parms parms;
  225. memset(&parms.orvals, 0, sizeof(parms.orvals));
  226. memset(&parms.andvals, 0xff, sizeof(parms.andvals));
  227. parms.orvals[cr] = 1 << bit;
  228. on_each_cpu(smp_ctl_bit_callback, &parms, 1);
  229. }
  230. EXPORT_SYMBOL(smp_ctl_set_bit);
  231. /*
  232. * Clear a bit in a control register of all cpus
  233. */
  234. void smp_ctl_clear_bit(int cr, int bit)
  235. {
  236. struct ec_creg_mask_parms parms;
  237. memset(&parms.orvals, 0, sizeof(parms.orvals));
  238. memset(&parms.andvals, 0xff, sizeof(parms.andvals));
  239. parms.andvals[cr] = ~(1L << bit);
  240. on_each_cpu(smp_ctl_bit_callback, &parms, 1);
  241. }
  242. EXPORT_SYMBOL(smp_ctl_clear_bit);
  243. #ifdef CONFIG_ZFCPDUMP
  244. static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
  245. {
  246. if (ipl_info.type != IPL_TYPE_FCP_DUMP)
  247. return;
  248. if (cpu >= NR_CPUS) {
  249. pr_warning("CPU %i exceeds the maximum %i and is excluded from "
  250. "the dump\n", cpu, NR_CPUS - 1);
  251. return;
  252. }
  253. zfcpdump_save_areas[cpu] = kmalloc(sizeof(struct save_area), GFP_KERNEL);
  254. while (raw_sigp(phy_cpu, sigp_stop_and_store_status) == sigp_busy)
  255. cpu_relax();
  256. memcpy(zfcpdump_save_areas[cpu],
  257. (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
  258. sizeof(struct save_area));
  259. }
  260. struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
  261. EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
  262. #else
  263. static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }
  264. #endif /* CONFIG_ZFCPDUMP */
  265. static int cpu_known(int cpu_id)
  266. {
  267. int cpu;
  268. for_each_present_cpu(cpu) {
  269. if (__cpu_logical_map[cpu] == cpu_id)
  270. return 1;
  271. }
  272. return 0;
  273. }
  274. static int smp_rescan_cpus_sigp(cpumask_t avail)
  275. {
  276. int cpu_id, logical_cpu;
  277. logical_cpu = cpumask_first(&avail);
  278. if (logical_cpu >= nr_cpu_ids)
  279. return 0;
  280. for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
  281. if (cpu_known(cpu_id))
  282. continue;
  283. __cpu_logical_map[logical_cpu] = cpu_id;
  284. smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
  285. if (!cpu_stopped(logical_cpu))
  286. continue;
  287. cpu_set(logical_cpu, cpu_present_map);
  288. smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
  289. logical_cpu = cpumask_next(logical_cpu, &avail);
  290. if (logical_cpu >= nr_cpu_ids)
  291. break;
  292. }
  293. return 0;
  294. }
  295. static int smp_rescan_cpus_sclp(cpumask_t avail)
  296. {
  297. struct sclp_cpu_info *info;
  298. int cpu_id, logical_cpu, cpu;
  299. int rc;
  300. logical_cpu = cpumask_first(&avail);
  301. if (logical_cpu >= nr_cpu_ids)
  302. return 0;
  303. info = kmalloc(sizeof(*info), GFP_KERNEL);
  304. if (!info)
  305. return -ENOMEM;
  306. rc = sclp_get_cpu_info(info);
  307. if (rc)
  308. goto out;
  309. for (cpu = 0; cpu < info->combined; cpu++) {
  310. if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
  311. continue;
  312. cpu_id = info->cpu[cpu].address;
  313. if (cpu_known(cpu_id))
  314. continue;
  315. __cpu_logical_map[logical_cpu] = cpu_id;
  316. smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
  317. cpu_set(logical_cpu, cpu_present_map);
  318. if (cpu >= info->configured)
  319. smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
  320. else
  321. smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
  322. logical_cpu = cpumask_next(logical_cpu, &avail);
  323. if (logical_cpu >= nr_cpu_ids)
  324. break;
  325. }
  326. out:
  327. kfree(info);
  328. return rc;
  329. }
  330. static int __smp_rescan_cpus(void)
  331. {
  332. cpumask_t avail;
  333. cpus_xor(avail, cpu_possible_map, cpu_present_map);
  334. if (smp_use_sigp_detection)
  335. return smp_rescan_cpus_sigp(avail);
  336. else
  337. return smp_rescan_cpus_sclp(avail);
  338. }
  339. static void __init smp_detect_cpus(void)
  340. {
  341. unsigned int cpu, c_cpus, s_cpus;
  342. struct sclp_cpu_info *info;
  343. u16 boot_cpu_addr, cpu_addr;
  344. c_cpus = 1;
  345. s_cpus = 0;
  346. boot_cpu_addr = __cpu_logical_map[0];
  347. info = kmalloc(sizeof(*info), GFP_KERNEL);
  348. if (!info)
  349. panic("smp_detect_cpus failed to allocate memory\n");
  350. /* Use sigp detection algorithm if sclp doesn't work. */
  351. if (sclp_get_cpu_info(info)) {
  352. smp_use_sigp_detection = 1;
  353. for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
  354. if (cpu == boot_cpu_addr)
  355. continue;
  356. if (!raw_cpu_stopped(cpu))
  357. continue;
  358. smp_get_save_area(c_cpus, cpu);
  359. c_cpus++;
  360. }
  361. goto out;
  362. }
  363. if (info->has_cpu_type) {
  364. for (cpu = 0; cpu < info->combined; cpu++) {
  365. if (info->cpu[cpu].address == boot_cpu_addr) {
  366. smp_cpu_type = info->cpu[cpu].type;
  367. break;
  368. }
  369. }
  370. }
  371. for (cpu = 0; cpu < info->combined; cpu++) {
  372. if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
  373. continue;
  374. cpu_addr = info->cpu[cpu].address;
  375. if (cpu_addr == boot_cpu_addr)
  376. continue;
  377. if (!raw_cpu_stopped(cpu_addr)) {
  378. s_cpus++;
  379. continue;
  380. }
  381. smp_get_save_area(c_cpus, cpu_addr);
  382. c_cpus++;
  383. }
  384. out:
  385. kfree(info);
  386. pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
  387. get_online_cpus();
  388. __smp_rescan_cpus();
  389. put_online_cpus();
  390. }
  391. /*
  392. * Activate a secondary processor.
  393. */
  394. int __cpuinit start_secondary(void *cpuvoid)
  395. {
  396. /* Setup the cpu */
  397. cpu_init();
  398. preempt_disable();
  399. /* Enable TOD clock interrupts on the secondary cpu. */
  400. init_cpu_timer();
  401. /* Enable cpu timer interrupts on the secondary cpu. */
  402. init_cpu_vtimer();
  403. /* Enable pfault pseudo page faults on this cpu. */
  404. pfault_init();
  405. /* call cpu notifiers */
  406. notify_cpu_starting(smp_processor_id());
  407. /* Mark this cpu as online */
  408. ipi_call_lock();
  409. cpu_set(smp_processor_id(), cpu_online_map);
  410. ipi_call_unlock();
  411. /* Switch on interrupts */
  412. local_irq_enable();
  413. /* Print info about this processor */
  414. print_cpu_info();
  415. /* cpu_idle will call schedule for us */
  416. cpu_idle();
  417. return 0;
  418. }
  419. static void __init smp_create_idle(unsigned int cpu)
  420. {
  421. struct task_struct *p;
  422. /*
  423. * don't care about the psw and regs settings since we'll never
  424. * reschedule the forked task.
  425. */
  426. p = fork_idle(cpu);
  427. if (IS_ERR(p))
  428. panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
  429. current_set[cpu] = p;
  430. }
  431. static int __cpuinit smp_alloc_lowcore(int cpu)
  432. {
  433. unsigned long async_stack, panic_stack;
  434. struct _lowcore *lowcore;
  435. lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
  436. if (!lowcore)
  437. return -ENOMEM;
  438. async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
  439. panic_stack = __get_free_page(GFP_KERNEL);
  440. if (!panic_stack || !async_stack)
  441. goto out;
  442. memcpy(lowcore, &S390_lowcore, 512);
  443. memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
  444. lowcore->async_stack = async_stack + ASYNC_SIZE;
  445. lowcore->panic_stack = panic_stack + PAGE_SIZE;
  446. #ifndef CONFIG_64BIT
  447. if (MACHINE_HAS_IEEE) {
  448. unsigned long save_area;
  449. save_area = get_zeroed_page(GFP_KERNEL);
  450. if (!save_area)
  451. goto out;
  452. lowcore->extended_save_area_addr = (u32) save_area;
  453. }
  454. #else
  455. if (vdso_alloc_per_cpu(cpu, lowcore))
  456. goto out;
  457. #endif
  458. lowcore_ptr[cpu] = lowcore;
  459. return 0;
  460. out:
  461. free_page(panic_stack);
  462. free_pages(async_stack, ASYNC_ORDER);
  463. free_pages((unsigned long) lowcore, LC_ORDER);
  464. return -ENOMEM;
  465. }
  466. static void smp_free_lowcore(int cpu)
  467. {
  468. struct _lowcore *lowcore;
  469. lowcore = lowcore_ptr[cpu];
  470. #ifndef CONFIG_64BIT
  471. if (MACHINE_HAS_IEEE)
  472. free_page((unsigned long) lowcore->extended_save_area_addr);
  473. #else
  474. vdso_free_per_cpu(cpu, lowcore);
  475. #endif
  476. free_page(lowcore->panic_stack - PAGE_SIZE);
  477. free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
  478. free_pages((unsigned long) lowcore, LC_ORDER);
  479. lowcore_ptr[cpu] = NULL;
  480. }
  481. /* Upping and downing of CPUs */
  482. int __cpuinit __cpu_up(unsigned int cpu)
  483. {
  484. struct _lowcore *cpu_lowcore;
  485. struct task_struct *idle;
  486. struct stack_frame *sf;
  487. u32 lowcore;
  488. int ccode;
  489. if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
  490. return -EIO;
  491. if (smp_alloc_lowcore(cpu))
  492. return -ENOMEM;
  493. do {
  494. ccode = sigp(cpu, sigp_initial_cpu_reset);
  495. if (ccode == sigp_busy)
  496. udelay(10);
  497. if (ccode == sigp_not_operational)
  498. goto err_out;
  499. } while (ccode == sigp_busy);
  500. lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
  501. while (sigp_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
  502. udelay(10);
  503. idle = current_set[cpu];
  504. cpu_lowcore = lowcore_ptr[cpu];
  505. cpu_lowcore->kernel_stack = (unsigned long)
  506. task_stack_page(idle) + THREAD_SIZE;
  507. cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
  508. sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
  509. - sizeof(struct pt_regs)
  510. - sizeof(struct stack_frame));
  511. memset(sf, 0, sizeof(struct stack_frame));
  512. sf->gprs[9] = (unsigned long) sf;
  513. cpu_lowcore->save_area[15] = (unsigned long) sf;
  514. __ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
  515. asm volatile(
  516. " stam 0,15,0(%0)"
  517. : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
  518. cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
  519. cpu_lowcore->current_task = (unsigned long) idle;
  520. cpu_lowcore->cpu_nr = cpu;
  521. cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
  522. cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
  523. cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
  524. eieio();
  525. while (sigp(cpu, sigp_restart) == sigp_busy)
  526. udelay(10);
  527. while (!cpu_online(cpu))
  528. cpu_relax();
  529. return 0;
  530. err_out:
  531. smp_free_lowcore(cpu);
  532. return -EIO;
  533. }
  534. static int __init setup_possible_cpus(char *s)
  535. {
  536. int pcpus, cpu;
  537. pcpus = simple_strtoul(s, NULL, 0);
  538. init_cpu_possible(cpumask_of(0));
  539. for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
  540. set_cpu_possible(cpu, true);
  541. return 0;
  542. }
  543. early_param("possible_cpus", setup_possible_cpus);
  544. #ifdef CONFIG_HOTPLUG_CPU
  545. int __cpu_disable(void)
  546. {
  547. struct ec_creg_mask_parms cr_parms;
  548. int cpu = smp_processor_id();
  549. cpu_clear(cpu, cpu_online_map);
  550. /* Disable pfault pseudo page faults on this cpu. */
  551. pfault_fini();
  552. memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
  553. memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
  554. /* disable all external interrupts */
  555. cr_parms.orvals[0] = 0;
  556. cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 |
  557. 1 << 11 | 1 << 10 | 1 << 6 | 1 << 4);
  558. /* disable all I/O interrupts */
  559. cr_parms.orvals[6] = 0;
  560. cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
  561. 1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
  562. /* disable most machine checks */
  563. cr_parms.orvals[14] = 0;
  564. cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
  565. 1 << 25 | 1 << 24);
  566. smp_ctl_bit_callback(&cr_parms);
  567. return 0;
  568. }
  569. void __cpu_die(unsigned int cpu)
  570. {
  571. /* Wait until target cpu is down */
  572. while (!cpu_stopped(cpu))
  573. cpu_relax();
  574. while (sigp_p(0, cpu, sigp_set_prefix) == sigp_busy)
  575. udelay(10);
  576. smp_free_lowcore(cpu);
  577. pr_info("Processor %d stopped\n", cpu);
  578. }
  579. void cpu_die(void)
  580. {
  581. idle_task_exit();
  582. while (sigp(smp_processor_id(), sigp_stop) == sigp_busy)
  583. cpu_relax();
  584. for (;;);
  585. }
  586. #endif /* CONFIG_HOTPLUG_CPU */
  587. void __init smp_prepare_cpus(unsigned int max_cpus)
  588. {
  589. #ifndef CONFIG_64BIT
  590. unsigned long save_area = 0;
  591. #endif
  592. unsigned long async_stack, panic_stack;
  593. struct _lowcore *lowcore;
  594. unsigned int cpu;
  595. smp_detect_cpus();
  596. /* request the 0x1201 emergency signal external interrupt */
  597. if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
  598. panic("Couldn't request external interrupt 0x1201");
  599. print_cpu_info();
  600. /* Reallocate current lowcore, but keep its contents. */
  601. lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
  602. panic_stack = __get_free_page(GFP_KERNEL);
  603. async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
  604. BUG_ON(!lowcore || !panic_stack || !async_stack);
  605. #ifndef CONFIG_64BIT
  606. if (MACHINE_HAS_IEEE)
  607. save_area = get_zeroed_page(GFP_KERNEL);
  608. #endif
  609. local_irq_disable();
  610. local_mcck_disable();
  611. lowcore_ptr[smp_processor_id()] = lowcore;
  612. *lowcore = S390_lowcore;
  613. lowcore->panic_stack = panic_stack + PAGE_SIZE;
  614. lowcore->async_stack = async_stack + ASYNC_SIZE;
  615. #ifndef CONFIG_64BIT
  616. if (MACHINE_HAS_IEEE)
  617. lowcore->extended_save_area_addr = (u32) save_area;
  618. #endif
  619. set_prefix((u32)(unsigned long) lowcore);
  620. local_mcck_enable();
  621. local_irq_enable();
  622. #ifdef CONFIG_64BIT
  623. if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
  624. BUG();
  625. #endif
  626. for_each_possible_cpu(cpu)
  627. if (cpu != smp_processor_id())
  628. smp_create_idle(cpu);
  629. }
  630. void __init smp_prepare_boot_cpu(void)
  631. {
  632. BUG_ON(smp_processor_id() != 0);
  633. current_thread_info()->cpu = 0;
  634. cpu_set(0, cpu_present_map);
  635. cpu_set(0, cpu_online_map);
  636. S390_lowcore.percpu_offset = __per_cpu_offset[0];
  637. current_set[0] = current;
  638. smp_cpu_state[0] = CPU_STATE_CONFIGURED;
  639. smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
  640. }
  641. void __init smp_cpus_done(unsigned int max_cpus)
  642. {
  643. }
  644. void __init smp_setup_processor_id(void)
  645. {
  646. S390_lowcore.cpu_nr = 0;
  647. __cpu_logical_map[0] = stap();
  648. }
  649. /*
  650. * the frequency of the profiling timer can be changed
  651. * by writing a multiplier value into /proc/profile.
  652. *
  653. * usually you want to run this on all CPUs ;)
  654. */
  655. int setup_profiling_timer(unsigned int multiplier)
  656. {
  657. return 0;
  658. }
  659. #ifdef CONFIG_HOTPLUG_CPU
  660. static ssize_t cpu_configure_show(struct sys_device *dev,
  661. struct sysdev_attribute *attr, char *buf)
  662. {
  663. ssize_t count;
  664. mutex_lock(&smp_cpu_state_mutex);
  665. count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
  666. mutex_unlock(&smp_cpu_state_mutex);
  667. return count;
  668. }
  669. static ssize_t cpu_configure_store(struct sys_device *dev,
  670. struct sysdev_attribute *attr,
  671. const char *buf, size_t count)
  672. {
  673. int cpu = dev->id;
  674. int val, rc;
  675. char delim;
  676. if (sscanf(buf, "%d %c", &val, &delim) != 1)
  677. return -EINVAL;
  678. if (val != 0 && val != 1)
  679. return -EINVAL;
  680. get_online_cpus();
  681. mutex_lock(&smp_cpu_state_mutex);
  682. rc = -EBUSY;
  683. /* disallow configuration changes of online cpus and cpu 0 */
  684. if (cpu_online(cpu) || cpu == 0)
  685. goto out;
  686. rc = 0;
  687. switch (val) {
  688. case 0:
  689. if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
  690. rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
  691. if (!rc) {
  692. smp_cpu_state[cpu] = CPU_STATE_STANDBY;
  693. smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
  694. }
  695. }
  696. break;
  697. case 1:
  698. if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
  699. rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
  700. if (!rc) {
  701. smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
  702. smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
  703. }
  704. }
  705. break;
  706. default:
  707. break;
  708. }
  709. out:
  710. mutex_unlock(&smp_cpu_state_mutex);
  711. put_online_cpus();
  712. return rc ? rc : count;
  713. }
  714. static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
  715. #endif /* CONFIG_HOTPLUG_CPU */
  716. static ssize_t cpu_polarization_show(struct sys_device *dev,
  717. struct sysdev_attribute *attr, char *buf)
  718. {
  719. int cpu = dev->id;
  720. ssize_t count;
  721. mutex_lock(&smp_cpu_state_mutex);
  722. switch (smp_cpu_polarization[cpu]) {
  723. case POLARIZATION_HRZ:
  724. count = sprintf(buf, "horizontal\n");
  725. break;
  726. case POLARIZATION_VL:
  727. count = sprintf(buf, "vertical:low\n");
  728. break;
  729. case POLARIZATION_VM:
  730. count = sprintf(buf, "vertical:medium\n");
  731. break;
  732. case POLARIZATION_VH:
  733. count = sprintf(buf, "vertical:high\n");
  734. break;
  735. default:
  736. count = sprintf(buf, "unknown\n");
  737. break;
  738. }
  739. mutex_unlock(&smp_cpu_state_mutex);
  740. return count;
  741. }
  742. static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);
  743. static ssize_t show_cpu_address(struct sys_device *dev,
  744. struct sysdev_attribute *attr, char *buf)
  745. {
  746. return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
  747. }
  748. static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);
  749. static struct attribute *cpu_common_attrs[] = {
  750. #ifdef CONFIG_HOTPLUG_CPU
  751. &attr_configure.attr,
  752. #endif
  753. &attr_address.attr,
  754. &attr_polarization.attr,
  755. NULL,
  756. };
  757. static struct attribute_group cpu_common_attr_group = {
  758. .attrs = cpu_common_attrs,
  759. };
  760. static ssize_t show_capability(struct sys_device *dev,
  761. struct sysdev_attribute *attr, char *buf)
  762. {
  763. unsigned int capability;
  764. int rc;
  765. rc = get_cpu_capability(&capability);
  766. if (rc)
  767. return rc;
  768. return sprintf(buf, "%u\n", capability);
  769. }
  770. static SYSDEV_ATTR(capability, 0444, show_capability, NULL);
  771. static ssize_t show_idle_count(struct sys_device *dev,
  772. struct sysdev_attribute *attr, char *buf)
  773. {
  774. struct s390_idle_data *idle;
  775. unsigned long long idle_count;
  776. unsigned int sequence;
  777. idle = &per_cpu(s390_idle, dev->id);
  778. repeat:
  779. sequence = idle->sequence;
  780. smp_rmb();
  781. if (sequence & 1)
  782. goto repeat;
  783. idle_count = idle->idle_count;
  784. if (idle->idle_enter)
  785. idle_count++;
  786. smp_rmb();
  787. if (idle->sequence != sequence)
  788. goto repeat;
  789. return sprintf(buf, "%llu\n", idle_count);
  790. }
  791. static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);
  792. static ssize_t show_idle_time(struct sys_device *dev,
  793. struct sysdev_attribute *attr, char *buf)
  794. {
  795. struct s390_idle_data *idle;
  796. unsigned long long now, idle_time, idle_enter;
  797. unsigned int sequence;
  798. idle = &per_cpu(s390_idle, dev->id);
  799. now = get_clock();
  800. repeat:
  801. sequence = idle->sequence;
  802. smp_rmb();
  803. if (sequence & 1)
  804. goto repeat;
  805. idle_time = idle->idle_time;
  806. idle_enter = idle->idle_enter;
  807. if (idle_enter != 0ULL && idle_enter < now)
  808. idle_time += now - idle_enter;
  809. smp_rmb();
  810. if (idle->sequence != sequence)
  811. goto repeat;
  812. return sprintf(buf, "%llu\n", idle_time >> 12);
  813. }
  814. static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
  815. static struct attribute *cpu_online_attrs[] = {
  816. &attr_capability.attr,
  817. &attr_idle_count.attr,
  818. &attr_idle_time_us.attr,
  819. NULL,
  820. };
  821. static struct attribute_group cpu_online_attr_group = {
  822. .attrs = cpu_online_attrs,
  823. };
  824. static int __cpuinit smp_cpu_notify(struct notifier_block *self,
  825. unsigned long action, void *hcpu)
  826. {
  827. unsigned int cpu = (unsigned int)(long)hcpu;
  828. struct cpu *c = &per_cpu(cpu_devices, cpu);
  829. struct sys_device *s = &c->sysdev;
  830. struct s390_idle_data *idle;
  831. switch (action) {
  832. case CPU_ONLINE:
  833. case CPU_ONLINE_FROZEN:
  834. idle = &per_cpu(s390_idle, cpu);
  835. memset(idle, 0, sizeof(struct s390_idle_data));
  836. if (sysfs_create_group(&s->kobj, &cpu_online_attr_group))
  837. return NOTIFY_BAD;
  838. break;
  839. case CPU_DEAD:
  840. case CPU_DEAD_FROZEN:
  841. sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
  842. break;
  843. }
  844. return NOTIFY_OK;
  845. }
  846. static struct notifier_block __cpuinitdata smp_cpu_nb = {
  847. .notifier_call = smp_cpu_notify,
  848. };
  849. static int __devinit smp_add_present_cpu(int cpu)
  850. {
  851. struct cpu *c = &per_cpu(cpu_devices, cpu);
  852. struct sys_device *s = &c->sysdev;
  853. int rc;
  854. c->hotpluggable = 1;
  855. rc = register_cpu(c, cpu);
  856. if (rc)
  857. goto out;
  858. rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
  859. if (rc)
  860. goto out_cpu;
  861. if (!cpu_online(cpu))
  862. goto out;
  863. rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
  864. if (!rc)
  865. return 0;
  866. sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
  867. out_cpu:
  868. #ifdef CONFIG_HOTPLUG_CPU
  869. unregister_cpu(c);
  870. #endif
  871. out:
  872. return rc;
  873. }
  874. #ifdef CONFIG_HOTPLUG_CPU
  875. int __ref smp_rescan_cpus(void)
  876. {
  877. cpumask_t newcpus;
  878. int cpu;
  879. int rc;
  880. get_online_cpus();
  881. mutex_lock(&smp_cpu_state_mutex);
  882. newcpus = cpu_present_map;
  883. rc = __smp_rescan_cpus();
  884. if (rc)
  885. goto out;
  886. cpus_andnot(newcpus, cpu_present_map, newcpus);
  887. for_each_cpu_mask(cpu, newcpus) {
  888. rc = smp_add_present_cpu(cpu);
  889. if (rc)
  890. cpu_clear(cpu, cpu_present_map);
  891. }
  892. rc = 0;
  893. out:
  894. mutex_unlock(&smp_cpu_state_mutex);
  895. put_online_cpus();
  896. if (!cpus_empty(newcpus))
  897. topology_schedule_update();
  898. return rc;
  899. }
  900. static ssize_t __ref rescan_store(struct sysdev_class *class,
  901. struct sysdev_class_attribute *attr,
  902. const char *buf,
  903. size_t count)
  904. {
  905. int rc;
  906. rc = smp_rescan_cpus();
  907. return rc ? rc : count;
  908. }
  909. static SYSDEV_CLASS_ATTR(rescan, 0200, NULL, rescan_store);
  910. #endif /* CONFIG_HOTPLUG_CPU */
  911. static ssize_t dispatching_show(struct sysdev_class *class,
  912. struct sysdev_class_attribute *attr,
  913. char *buf)
  914. {
  915. ssize_t count;
  916. mutex_lock(&smp_cpu_state_mutex);
  917. count = sprintf(buf, "%d\n", cpu_management);
  918. mutex_unlock(&smp_cpu_state_mutex);
  919. return count;
  920. }
  921. static ssize_t dispatching_store(struct sysdev_class *dev,
  922. struct sysdev_class_attribute *attr,
  923. const char *buf,
  924. size_t count)
  925. {
  926. int val, rc;
  927. char delim;
  928. if (sscanf(buf, "%d %c", &val, &delim) != 1)
  929. return -EINVAL;
  930. if (val != 0 && val != 1)
  931. return -EINVAL;
  932. rc = 0;
  933. get_online_cpus();
  934. mutex_lock(&smp_cpu_state_mutex);
  935. if (cpu_management == val)
  936. goto out;
  937. rc = topology_set_cpu_management(val);
  938. if (!rc)
  939. cpu_management = val;
  940. out:
  941. mutex_unlock(&smp_cpu_state_mutex);
  942. put_online_cpus();
  943. return rc ? rc : count;
  944. }
  945. static SYSDEV_CLASS_ATTR(dispatching, 0644, dispatching_show,
  946. dispatching_store);
  947. static int __init topology_init(void)
  948. {
  949. int cpu;
  950. int rc;
  951. register_cpu_notifier(&smp_cpu_nb);
  952. #ifdef CONFIG_HOTPLUG_CPU
  953. rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_rescan);
  954. if (rc)
  955. return rc;
  956. #endif
  957. rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_dispatching);
  958. if (rc)
  959. return rc;
  960. for_each_present_cpu(cpu) {
  961. rc = smp_add_present_cpu(cpu);
  962. if (rc)
  963. return rc;
  964. }
  965. return 0;
  966. }
  967. subsys_initcall(topology_init);