smtc.c 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528
  1. /*
  2. * This program is free software; you can redistribute it and/or
  3. * modify it under the terms of the GNU General Public License
  4. * as published by the Free Software Foundation; either version 2
  5. * of the License, or (at your option) any later version.
  6. *
  7. * This program is distributed in the hope that it will be useful,
  8. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. * GNU General Public License for more details.
  11. *
  12. * You should have received a copy of the GNU General Public License
  13. * along with this program; if not, write to the Free Software
  14. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15. *
  16. * Copyright (C) 2004 Mips Technologies, Inc
  17. * Copyright (C) 2008 Kevin D. Kissell
  18. */
  19. #include <linux/clockchips.h>
  20. #include <linux/kernel.h>
  21. #include <linux/sched.h>
  22. #include <linux/smp.h>
  23. #include <linux/cpumask.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/kernel_stat.h>
  26. #include <linux/module.h>
  27. #include <linux/ftrace.h>
  28. #include <linux/slab.h>
  29. #include <asm/cpu.h>
  30. #include <asm/processor.h>
  31. #include <linux/atomic.h>
  32. #include <asm/hardirq.h>
  33. #include <asm/hazards.h>
  34. #include <asm/irq.h>
  35. #include <asm/idle.h>
  36. #include <asm/mmu_context.h>
  37. #include <asm/mipsregs.h>
  38. #include <asm/cacheflush.h>
  39. #include <asm/time.h>
  40. #include <asm/addrspace.h>
  41. #include <asm/smtc.h>
  42. #include <asm/smtc_proc.h>
  43. #include <asm/setup.h>
  44. /*
  45. * SMTC Kernel needs to manipulate low-level CPU interrupt mask
  46. * in do_IRQ. These are passed in setup_irq_smtc() and stored
  47. * in this table.
  48. */
  49. unsigned long irq_hwmask[NR_IRQS];
  50. #define LOCK_MT_PRA() \
  51. local_irq_save(flags); \
  52. mtflags = dmt()
  53. #define UNLOCK_MT_PRA() \
  54. emt(mtflags); \
  55. local_irq_restore(flags)
  56. #define LOCK_CORE_PRA() \
  57. local_irq_save(flags); \
  58. mtflags = dvpe()
  59. #define UNLOCK_CORE_PRA() \
  60. evpe(mtflags); \
  61. local_irq_restore(flags)
  62. /*
  63. * Data structures purely associated with SMTC parallelism
  64. */
  65. /*
  66. * Table for tracking ASIDs whose lifetime is prolonged.
  67. */
  68. asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS];
  69. /*
  70. * Number of InterProcessor Interrupt (IPI) message buffers to allocate
  71. */
  72. #define IPIBUF_PER_CPU 4
  73. struct smtc_ipi_q IPIQ[NR_CPUS];
  74. static struct smtc_ipi_q freeIPIq;
  75. /*
  76. * Number of FPU contexts for each VPE
  77. */
  78. static int smtc_nconf1[MAX_SMTC_VPES];
  79. /* Forward declarations */
  80. void ipi_decode(struct smtc_ipi *);
  81. static void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
  82. static void setup_cross_vpe_interrupts(unsigned int nvpe);
  83. void init_smtc_stats(void);
  84. /* Global SMTC Status */
  85. unsigned int smtc_status;
  86. /* Boot command line configuration overrides */
  87. static int vpe0limit;
  88. static int ipibuffers;
  89. static int nostlb;
  90. static int asidmask;
  91. unsigned long smtc_asid_mask = 0xff;
  92. static int __init vpe0tcs(char *str)
  93. {
  94. get_option(&str, &vpe0limit);
  95. return 1;
  96. }
  97. static int __init ipibufs(char *str)
  98. {
  99. get_option(&str, &ipibuffers);
  100. return 1;
  101. }
  102. static int __init stlb_disable(char *s)
  103. {
  104. nostlb = 1;
  105. return 1;
  106. }
  107. static int __init asidmask_set(char *str)
  108. {
  109. get_option(&str, &asidmask);
  110. switch (asidmask) {
  111. case 0x1:
  112. case 0x3:
  113. case 0x7:
  114. case 0xf:
  115. case 0x1f:
  116. case 0x3f:
  117. case 0x7f:
  118. case 0xff:
  119. smtc_asid_mask = (unsigned long)asidmask;
  120. break;
  121. default:
  122. printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask);
  123. }
  124. return 1;
  125. }
  126. __setup("vpe0tcs=", vpe0tcs);
  127. __setup("ipibufs=", ipibufs);
  128. __setup("nostlb", stlb_disable);
  129. __setup("asidmask=", asidmask_set);
  130. #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
  131. static int hang_trig;
  132. static int __init hangtrig_enable(char *s)
  133. {
  134. hang_trig = 1;
  135. return 1;
  136. }
  137. __setup("hangtrig", hangtrig_enable);
  138. #define DEFAULT_BLOCKED_IPI_LIMIT 32
  139. static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT;
  140. static int __init tintq(char *str)
  141. {
  142. get_option(&str, &timerq_limit);
  143. return 1;
  144. }
  145. __setup("tintq=", tintq);
  146. static int imstuckcount[MAX_SMTC_VPES][8];
  147. /* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */
  148. static int vpemask[MAX_SMTC_VPES][8] = {
  149. {0, 0, 1, 0, 0, 0, 0, 1},
  150. {0, 0, 0, 0, 0, 0, 0, 1}
  151. };
  152. int tcnoprog[NR_CPUS];
  153. static atomic_t idle_hook_initialized = ATOMIC_INIT(0);
  154. static int clock_hang_reported[NR_CPUS];
  155. #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
  156. /*
  157. * Configure shared TLB - VPC configuration bit must be set by caller
  158. */
  159. static void smtc_configure_tlb(void)
  160. {
  161. int i, tlbsiz, vpes;
  162. unsigned long mvpconf0;
  163. unsigned long config1val;
  164. /* Set up ASID preservation table */
  165. for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) {
  166. for(i = 0; i < MAX_SMTC_ASIDS; i++) {
  167. smtc_live_asid[vpes][i] = 0;
  168. }
  169. }
  170. mvpconf0 = read_c0_mvpconf0();
  171. if ((vpes = ((mvpconf0 & MVPCONF0_PVPE)
  172. >> MVPCONF0_PVPE_SHIFT) + 1) > 1) {
  173. /* If we have multiple VPEs, try to share the TLB */
  174. if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) {
  175. /*
  176. * If TLB sizing is programmable, shared TLB
  177. * size is the total available complement.
  178. * Otherwise, we have to take the sum of all
  179. * static VPE TLB entries.
  180. */
  181. if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE)
  182. >> MVPCONF0_PTLBE_SHIFT)) == 0) {
  183. /*
  184. * If there's more than one VPE, there had better
  185. * be more than one TC, because we need one to bind
  186. * to each VPE in turn to be able to read
  187. * its configuration state!
  188. */
  189. settc(1);
  190. /* Stop the TC from doing anything foolish */
  191. write_tc_c0_tchalt(TCHALT_H);
  192. mips_ihb();
  193. /* No need to un-Halt - that happens later anyway */
  194. for (i=0; i < vpes; i++) {
  195. write_tc_c0_tcbind(i);
  196. /*
  197. * To be 100% sure we're really getting the right
  198. * information, we exit the configuration state
  199. * and do an IHB after each rebinding.
  200. */
  201. write_c0_mvpcontrol(
  202. read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
  203. mips_ihb();
  204. /*
  205. * Only count if the MMU Type indicated is TLB
  206. */
  207. if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) {
  208. config1val = read_vpe_c0_config1();
  209. tlbsiz += ((config1val >> 25) & 0x3f) + 1;
  210. }
  211. /* Put core back in configuration state */
  212. write_c0_mvpcontrol(
  213. read_c0_mvpcontrol() | MVPCONTROL_VPC );
  214. mips_ihb();
  215. }
  216. }
  217. write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB);
  218. ehb();
  219. /*
  220. * Setup kernel data structures to use software total,
  221. * rather than read the per-VPE Config1 value. The values
  222. * for "CPU 0" gets copied to all the other CPUs as part
  223. * of their initialization in smtc_cpu_setup().
  224. */
  225. /* MIPS32 limits TLB indices to 64 */
  226. if (tlbsiz > 64)
  227. tlbsiz = 64;
  228. cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz;
  229. smtc_status |= SMTC_TLB_SHARED;
  230. local_flush_tlb_all();
  231. printk("TLB of %d entry pairs shared by %d VPEs\n",
  232. tlbsiz, vpes);
  233. } else {
  234. printk("WARNING: TLB Not Sharable on SMTC Boot!\n");
  235. }
  236. }
  237. }
  238. /*
  239. * Incrementally build the CPU map out of constituent MIPS MT cores,
  240. * using the specified available VPEs and TCs. Plaform code needs
  241. * to ensure that each MIPS MT core invokes this routine on reset,
  242. * one at a time(!).
  243. *
  244. * This version of the build_cpu_map and prepare_cpus routines assumes
  245. * that *all* TCs of a MIPS MT core will be used for Linux, and that
  246. * they will be spread across *all* available VPEs (to minimise the
  247. * loss of efficiency due to exception service serialization).
  248. * An improved version would pick up configuration information and
  249. * possibly leave some TCs/VPEs as "slave" processors.
  250. *
  251. * Use c0_MVPConf0 to find out how many TCs are available, setting up
  252. * cpu_possible_mask and the logical/physical mappings.
  253. */
  254. int __init smtc_build_cpu_map(int start_cpu_slot)
  255. {
  256. int i, ntcs;
  257. /*
  258. * The CPU map isn't actually used for anything at this point,
  259. * so it's not clear what else we should do apart from set
  260. * everything up so that "logical" = "physical".
  261. */
  262. ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
  263. for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) {
  264. set_cpu_possible(i, true);
  265. __cpu_number_map[i] = i;
  266. __cpu_logical_map[i] = i;
  267. }
  268. #ifdef CONFIG_MIPS_MT_FPAFF
  269. /* Initialize map of CPUs with FPUs */
  270. cpus_clear(mt_fpu_cpumask);
  271. #endif
  272. /* One of those TC's is the one booting, and not a secondary... */
  273. printk("%i available secondary CPU TC(s)\n", i - 1);
  274. return i;
  275. }
  276. /*
  277. * Common setup before any secondaries are started
  278. * Make sure all CPUs are in a sensible state before we boot any of the
  279. * secondaries.
  280. *
  281. * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly
  282. * as possible across the available VPEs.
  283. */
  284. static void smtc_tc_setup(int vpe, int tc, int cpu)
  285. {
  286. static int cp1contexts[MAX_SMTC_VPES];
  287. /*
  288. * Make a local copy of the available FPU contexts in order
  289. * to keep track of TCs that can have one.
  290. */
  291. if (tc == 1)
  292. {
  293. /*
  294. * FIXME: Multi-core SMTC hasn't been tested and the
  295. * maximum number of VPEs may change.
  296. */
  297. cp1contexts[0] = smtc_nconf1[0] - 1;
  298. cp1contexts[1] = smtc_nconf1[1];
  299. }
  300. settc(tc);
  301. write_tc_c0_tchalt(TCHALT_H);
  302. mips_ihb();
  303. write_tc_c0_tcstatus((read_tc_c0_tcstatus()
  304. & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT))
  305. | TCSTATUS_A);
  306. /*
  307. * TCContext gets an offset from the base of the IPIQ array
  308. * to be used in low-level code to detect the presence of
  309. * an active IPI queue.
  310. */
  311. write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16);
  312. /* Bind TC to VPE. */
  313. write_tc_c0_tcbind(vpe);
  314. /* In general, all TCs should have the same cpu_data indications. */
  315. memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips));
  316. /* Check to see if there is a FPU context available for this TC. */
  317. if (!cp1contexts[vpe])
  318. cpu_data[cpu].options &= ~MIPS_CPU_FPU;
  319. else
  320. cp1contexts[vpe]--;
  321. /* Store the TC and VPE into the cpu_data structure. */
  322. cpu_data[cpu].vpe_id = vpe;
  323. cpu_data[cpu].tc_id = tc;
  324. /* FIXME: Multi-core SMTC hasn't been tested, but be prepared. */
  325. cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff;
  326. }
  327. /*
  328. * Tweak to get Count registers synced as closely as possible. The
  329. * value seems good for 34K-class cores.
  330. */
  331. #define CP0_SKEW 8
  332. void smtc_prepare_cpus(int cpus)
  333. {
  334. int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu;
  335. unsigned long flags;
  336. unsigned long val;
  337. int nipi;
  338. struct smtc_ipi *pipi;
  339. /* disable interrupts so we can disable MT */
  340. local_irq_save(flags);
  341. /* disable MT so we can configure */
  342. dvpe();
  343. dmt();
  344. spin_lock_init(&freeIPIq.lock);
  345. /*
  346. * We probably don't have as many VPEs as we do SMP "CPUs",
  347. * but it's possible - and in any case we'll never use more!
  348. */
  349. for (i=0; i<NR_CPUS; i++) {
  350. IPIQ[i].head = IPIQ[i].tail = NULL;
  351. spin_lock_init(&IPIQ[i].lock);
  352. IPIQ[i].depth = 0;
  353. IPIQ[i].resched_flag = 0; /* No reschedules queued initially */
  354. }
  355. /* cpu_data index starts at zero */
  356. cpu = 0;
  357. cpu_data[cpu].vpe_id = 0;
  358. cpu_data[cpu].tc_id = 0;
  359. cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff;
  360. cpu++;
  361. /* Report on boot-time options */
  362. mips_mt_set_cpuoptions();
  363. if (vpelimit > 0)
  364. printk("Limit of %d VPEs set\n", vpelimit);
  365. if (tclimit > 0)
  366. printk("Limit of %d TCs set\n", tclimit);
  367. if (nostlb) {
  368. printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n");
  369. }
  370. if (asidmask)
  371. printk("ASID mask value override to 0x%x\n", asidmask);
  372. /* Temporary */
  373. #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
  374. if (hang_trig)
  375. printk("Logic Analyser Trigger on suspected TC hang\n");
  376. #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
  377. /* Put MVPE's into 'configuration state' */
  378. write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC );
  379. val = read_c0_mvpconf0();
  380. nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1;
  381. if (vpelimit > 0 && nvpe > vpelimit)
  382. nvpe = vpelimit;
  383. ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
  384. if (ntc > NR_CPUS)
  385. ntc = NR_CPUS;
  386. if (tclimit > 0 && ntc > tclimit)
  387. ntc = tclimit;
  388. slop = ntc % nvpe;
  389. for (i = 0; i < nvpe; i++) {
  390. tcpervpe[i] = ntc / nvpe;
  391. if (slop) {
  392. if((slop - i) > 0) tcpervpe[i]++;
  393. }
  394. }
  395. /* Handle command line override for VPE0 */
  396. if (vpe0limit > ntc) vpe0limit = ntc;
  397. if (vpe0limit > 0) {
  398. int slopslop;
  399. if (vpe0limit < tcpervpe[0]) {
  400. /* Reducing TC count - distribute to others */
  401. slop = tcpervpe[0] - vpe0limit;
  402. slopslop = slop % (nvpe - 1);
  403. tcpervpe[0] = vpe0limit;
  404. for (i = 1; i < nvpe; i++) {
  405. tcpervpe[i] += slop / (nvpe - 1);
  406. if(slopslop && ((slopslop - (i - 1) > 0)))
  407. tcpervpe[i]++;
  408. }
  409. } else if (vpe0limit > tcpervpe[0]) {
  410. /* Increasing TC count - steal from others */
  411. slop = vpe0limit - tcpervpe[0];
  412. slopslop = slop % (nvpe - 1);
  413. tcpervpe[0] = vpe0limit;
  414. for (i = 1; i < nvpe; i++) {
  415. tcpervpe[i] -= slop / (nvpe - 1);
  416. if(slopslop && ((slopslop - (i - 1) > 0)))
  417. tcpervpe[i]--;
  418. }
  419. }
  420. }
  421. /* Set up shared TLB */
  422. smtc_configure_tlb();
  423. for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) {
  424. /* Get number of CP1 contexts for each VPE. */
  425. if (tc == 0)
  426. {
  427. /*
  428. * Do not call settc() for TC0 or the FPU context
  429. * value will be incorrect. Besides, we know that
  430. * we are TC0 anyway.
  431. */
  432. smtc_nconf1[0] = ((read_vpe_c0_vpeconf1() &
  433. VPECONF1_NCP1) >> VPECONF1_NCP1_SHIFT);
  434. if (nvpe == 2)
  435. {
  436. settc(1);
  437. smtc_nconf1[1] = ((read_vpe_c0_vpeconf1() &
  438. VPECONF1_NCP1) >> VPECONF1_NCP1_SHIFT);
  439. settc(0);
  440. }
  441. }
  442. if (tcpervpe[vpe] == 0)
  443. continue;
  444. if (vpe != 0)
  445. printk(", ");
  446. printk("VPE %d: TC", vpe);
  447. for (i = 0; i < tcpervpe[vpe]; i++) {
  448. /*
  449. * TC 0 is bound to VPE 0 at reset,
  450. * and is presumably executing this
  451. * code. Leave it alone!
  452. */
  453. if (tc != 0) {
  454. smtc_tc_setup(vpe, tc, cpu);
  455. if (vpe != 0) {
  456. /*
  457. * Set MVP bit (possibly again). Do it
  458. * here to catch CPUs that have no TCs
  459. * bound to the VPE at reset. In that
  460. * case, a TC must be bound to the VPE
  461. * before we can set VPEControl[MVP]
  462. */
  463. write_vpe_c0_vpeconf0(
  464. read_vpe_c0_vpeconf0() |
  465. VPECONF0_MVP);
  466. }
  467. cpu++;
  468. }
  469. printk(" %d", tc);
  470. tc++;
  471. }
  472. if (vpe != 0) {
  473. /*
  474. * Allow this VPE to control others.
  475. */
  476. write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() |
  477. VPECONF0_MVP);
  478. /*
  479. * Clear any stale software interrupts from VPE's Cause
  480. */
  481. write_vpe_c0_cause(0);
  482. /*
  483. * Clear ERL/EXL of VPEs other than 0
  484. * and set restricted interrupt enable/mask.
  485. */
  486. write_vpe_c0_status((read_vpe_c0_status()
  487. & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM))
  488. | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7
  489. | ST0_IE));
  490. /*
  491. * set config to be the same as vpe0,
  492. * particularly kseg0 coherency alg
  493. */
  494. write_vpe_c0_config(read_c0_config());
  495. /* Clear any pending timer interrupt */
  496. write_vpe_c0_compare(0);
  497. /* Propagate Config7 */
  498. write_vpe_c0_config7(read_c0_config7());
  499. write_vpe_c0_count(read_c0_count() + CP0_SKEW);
  500. ehb();
  501. }
  502. /* enable multi-threading within VPE */
  503. write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE);
  504. /* enable the VPE */
  505. write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA);
  506. }
  507. /*
  508. * Pull any physically present but unused TCs out of circulation.
  509. */
  510. while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) {
  511. set_cpu_possible(tc, false);
  512. set_cpu_present(tc, false);
  513. tc++;
  514. }
  515. /* release config state */
  516. write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
  517. printk("\n");
  518. /* Set up coprocessor affinity CPU mask(s) */
  519. #ifdef CONFIG_MIPS_MT_FPAFF
  520. for (tc = 0; tc < ntc; tc++) {
  521. if (cpu_data[tc].options & MIPS_CPU_FPU)
  522. cpu_set(tc, mt_fpu_cpumask);
  523. }
  524. #endif
  525. /* set up ipi interrupts... */
  526. /* If we have multiple VPEs running, set up the cross-VPE interrupt */
  527. setup_cross_vpe_interrupts(nvpe);
  528. /* Set up queue of free IPI "messages". */
  529. nipi = NR_CPUS * IPIBUF_PER_CPU;
  530. if (ipibuffers > 0)
  531. nipi = ipibuffers;
  532. pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL);
  533. if (pipi == NULL)
  534. panic("kmalloc of IPI message buffers failed");
  535. else
  536. printk("IPI buffer pool of %d buffers\n", nipi);
  537. for (i = 0; i < nipi; i++) {
  538. smtc_ipi_nq(&freeIPIq, pipi);
  539. pipi++;
  540. }
  541. /* Arm multithreading and enable other VPEs - but all TCs are Halted */
  542. emt(EMT_ENABLE);
  543. evpe(EVPE_ENABLE);
  544. local_irq_restore(flags);
  545. /* Initialize SMTC /proc statistics/diagnostics */
  546. init_smtc_stats();
  547. }
  548. /*
  549. * Setup the PC, SP, and GP of a secondary processor and start it
  550. * running!
  551. * smp_bootstrap is the place to resume from
  552. * __KSTK_TOS(idle) is apparently the stack pointer
  553. * (unsigned long)idle->thread_info the gp
  554. *
  555. */
  556. void smtc_boot_secondary(int cpu, struct task_struct *idle)
  557. {
  558. extern u32 kernelsp[NR_CPUS];
  559. unsigned long flags;
  560. int mtflags;
  561. LOCK_MT_PRA();
  562. if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
  563. dvpe();
  564. }
  565. settc(cpu_data[cpu].tc_id);
  566. /* pc */
  567. write_tc_c0_tcrestart((unsigned long)&smp_bootstrap);
  568. /* stack pointer */
  569. kernelsp[cpu] = __KSTK_TOS(idle);
  570. write_tc_gpr_sp(__KSTK_TOS(idle));
  571. /* global pointer */
  572. write_tc_gpr_gp((unsigned long)task_thread_info(idle));
  573. smtc_status |= SMTC_MTC_ACTIVE;
  574. write_tc_c0_tchalt(0);
  575. if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
  576. evpe(EVPE_ENABLE);
  577. }
  578. UNLOCK_MT_PRA();
  579. }
  580. void smtc_init_secondary(void)
  581. {
  582. }
  583. void smtc_smp_finish(void)
  584. {
  585. int cpu = smp_processor_id();
  586. /*
  587. * Lowest-numbered CPU per VPE starts a clock tick.
  588. * Like per_cpu_trap_init() hack, this assumes that
  589. * SMTC init code assigns TCs consdecutively and
  590. * in ascending order across available VPEs.
  591. */
  592. if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id))
  593. write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ);
  594. local_irq_enable();
  595. printk("TC %d going on-line as CPU %d\n",
  596. cpu_data[smp_processor_id()].tc_id, smp_processor_id());
  597. }
  598. void smtc_cpus_done(void)
  599. {
  600. }
  601. /*
  602. * Support for SMTC-optimized driver IRQ registration
  603. */
  604. /*
  605. * SMTC Kernel needs to manipulate low-level CPU interrupt mask
  606. * in do_IRQ. These are passed in setup_irq_smtc() and stored
  607. * in this table.
  608. */
  609. int setup_irq_smtc(unsigned int irq, struct irqaction * new,
  610. unsigned long hwmask)
  611. {
  612. #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
  613. unsigned int vpe = current_cpu_data.vpe_id;
  614. vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1;
  615. #endif
  616. irq_hwmask[irq] = hwmask;
  617. return setup_irq(irq, new);
  618. }
  619. #ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
  620. /*
  621. * Support for IRQ affinity to TCs
  622. */
  623. void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity)
  624. {
  625. /*
  626. * If a "fast path" cache of quickly decodable affinity state
  627. * is maintained, this is where it gets done, on a call up
  628. * from the platform affinity code.
  629. */
  630. }
  631. void smtc_forward_irq(struct irq_data *d)
  632. {
  633. unsigned int irq = d->irq;
  634. int target;
  635. /*
  636. * OK wise guy, now figure out how to get the IRQ
  637. * to be serviced on an authorized "CPU".
  638. *
  639. * Ideally, to handle the situation where an IRQ has multiple
  640. * eligible CPUS, we would maintain state per IRQ that would
  641. * allow a fair distribution of service requests. Since the
  642. * expected use model is any-or-only-one, for simplicity
  643. * and efficiency, we just pick the easiest one to find.
  644. */
  645. target = cpumask_first(d->affinity);
  646. /*
  647. * We depend on the platform code to have correctly processed
  648. * IRQ affinity change requests to ensure that the IRQ affinity
  649. * mask has been purged of bits corresponding to nonexistent and
  650. * offline "CPUs", and to TCs bound to VPEs other than the VPE
  651. * connected to the physical interrupt input for the interrupt
  652. * in question. Otherwise we have a nasty problem with interrupt
  653. * mask management. This is best handled in non-performance-critical
  654. * platform IRQ affinity setting code, to minimize interrupt-time
  655. * checks.
  656. */
  657. /* If no one is eligible, service locally */
  658. if (target >= NR_CPUS)
  659. do_IRQ_no_affinity(irq);
  660. else
  661. smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq);
  662. }
  663. #endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */
  664. /*
  665. * IPI model for SMTC is tricky, because interrupts aren't TC-specific.
  666. * Within a VPE one TC can interrupt another by different approaches.
  667. * The easiest to get right would probably be to make all TCs except
  668. * the target IXMT and set a software interrupt, but an IXMT-based
  669. * scheme requires that a handler must run before a new IPI could
  670. * be sent, which would break the "broadcast" loops in MIPS MT.
  671. * A more gonzo approach within a VPE is to halt the TC, extract
  672. * its Restart, Status, and a couple of GPRs, and program the Restart
  673. * address to emulate an interrupt.
  674. *
  675. * Within a VPE, one can be confident that the target TC isn't in
  676. * a critical EXL state when halted, since the write to the Halt
  677. * register could not have issued on the writing thread if the
  678. * halting thread had EXL set. So k0 and k1 of the target TC
  679. * can be used by the injection code. Across VPEs, one can't
  680. * be certain that the target TC isn't in a critical exception
  681. * state. So we try a two-step process of sending a software
  682. * interrupt to the target VPE, which either handles the event
  683. * itself (if it was the target) or injects the event within
  684. * the VPE.
  685. */
  686. static void smtc_ipi_qdump(void)
  687. {
  688. int i;
  689. struct smtc_ipi *temp;
  690. for (i = 0; i < NR_CPUS ;i++) {
  691. pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
  692. i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail,
  693. IPIQ[i].depth);
  694. temp = IPIQ[i].head;
  695. while (temp != IPIQ[i].tail) {
  696. pr_debug("%d %d %d: ", temp->type, temp->dest,
  697. (int)temp->arg);
  698. #ifdef SMTC_IPI_DEBUG
  699. pr_debug("%u %lu\n", temp->sender, temp->stamp);
  700. #else
  701. pr_debug("\n");
  702. #endif
  703. temp = temp->flink;
  704. }
  705. }
  706. }
  707. /*
  708. * The standard atomic.h primitives don't quite do what we want
  709. * here: We need an atomic add-and-return-previous-value (which
  710. * could be done with atomic_add_return and a decrement) and an
  711. * atomic set/zero-and-return-previous-value (which can't really
  712. * be done with the atomic.h primitives). And since this is
  713. * MIPS MT, we can assume that we have LL/SC.
  714. */
  715. static inline int atomic_postincrement(atomic_t *v)
  716. {
  717. unsigned long result;
  718. unsigned long temp;
  719. __asm__ __volatile__(
  720. "1: ll %0, %2 \n"
  721. " addu %1, %0, 1 \n"
  722. " sc %1, %2 \n"
  723. " beqz %1, 1b \n"
  724. __WEAK_LLSC_MB
  725. : "=&r" (result), "=&r" (temp), "=m" (v->counter)
  726. : "m" (v->counter)
  727. : "memory");
  728. return result;
  729. }
  730. void smtc_send_ipi(int cpu, int type, unsigned int action)
  731. {
  732. int tcstatus;
  733. struct smtc_ipi *pipi;
  734. unsigned long flags;
  735. int mtflags;
  736. unsigned long tcrestart;
  737. int set_resched_flag = (type == LINUX_SMP_IPI &&
  738. action == SMP_RESCHEDULE_YOURSELF);
  739. if (cpu == smp_processor_id()) {
  740. printk("Cannot Send IPI to self!\n");
  741. return;
  742. }
  743. if (set_resched_flag && IPIQ[cpu].resched_flag != 0)
  744. return; /* There is a reschedule queued already */
  745. /* Set up a descriptor, to be delivered either promptly or queued */
  746. pipi = smtc_ipi_dq(&freeIPIq);
  747. if (pipi == NULL) {
  748. bust_spinlocks(1);
  749. mips_mt_regdump(dvpe());
  750. panic("IPI Msg. Buffers Depleted");
  751. }
  752. pipi->type = type;
  753. pipi->arg = (void *)action;
  754. pipi->dest = cpu;
  755. if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
  756. /* If not on same VPE, enqueue and send cross-VPE interrupt */
  757. IPIQ[cpu].resched_flag |= set_resched_flag;
  758. smtc_ipi_nq(&IPIQ[cpu], pipi);
  759. LOCK_CORE_PRA();
  760. settc(cpu_data[cpu].tc_id);
  761. write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1);
  762. UNLOCK_CORE_PRA();
  763. } else {
  764. /*
  765. * Not sufficient to do a LOCK_MT_PRA (dmt) here,
  766. * since ASID shootdown on the other VPE may
  767. * collide with this operation.
  768. */
  769. LOCK_CORE_PRA();
  770. settc(cpu_data[cpu].tc_id);
  771. /* Halt the targeted TC */
  772. write_tc_c0_tchalt(TCHALT_H);
  773. mips_ihb();
  774. /*
  775. * Inspect TCStatus - if IXMT is set, we have to queue
  776. * a message. Otherwise, we set up the "interrupt"
  777. * of the other TC
  778. */
  779. tcstatus = read_tc_c0_tcstatus();
  780. if ((tcstatus & TCSTATUS_IXMT) != 0) {
  781. /*
  782. * If we're in the the irq-off version of the wait
  783. * loop, we need to force exit from the wait and
  784. * do a direct post of the IPI.
  785. */
  786. if (cpu_wait == r4k_wait_irqoff) {
  787. tcrestart = read_tc_c0_tcrestart();
  788. if (address_is_in_r4k_wait_irqoff(tcrestart)) {
  789. write_tc_c0_tcrestart(__pastwait);
  790. tcstatus &= ~TCSTATUS_IXMT;
  791. write_tc_c0_tcstatus(tcstatus);
  792. goto postdirect;
  793. }
  794. }
  795. /*
  796. * Otherwise we queue the message for the target TC
  797. * to pick up when he does a local_irq_restore()
  798. */
  799. write_tc_c0_tchalt(0);
  800. UNLOCK_CORE_PRA();
  801. IPIQ[cpu].resched_flag |= set_resched_flag;
  802. smtc_ipi_nq(&IPIQ[cpu], pipi);
  803. } else {
  804. postdirect:
  805. post_direct_ipi(cpu, pipi);
  806. write_tc_c0_tchalt(0);
  807. UNLOCK_CORE_PRA();
  808. }
  809. }
  810. }
  811. /*
  812. * Send IPI message to Halted TC, TargTC/TargVPE already having been set
  813. */
  814. static void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
  815. {
  816. struct pt_regs *kstack;
  817. unsigned long tcstatus;
  818. unsigned long tcrestart;
  819. extern u32 kernelsp[NR_CPUS];
  820. extern void __smtc_ipi_vector(void);
  821. //printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu);
  822. /* Extract Status, EPC from halted TC */
  823. tcstatus = read_tc_c0_tcstatus();
  824. tcrestart = read_tc_c0_tcrestart();
  825. /* If TCRestart indicates a WAIT instruction, advance the PC */
  826. if ((tcrestart & 0x80000000)
  827. && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) {
  828. tcrestart += 4;
  829. }
  830. /*
  831. * Save on TC's future kernel stack
  832. *
  833. * CU bit of Status is indicator that TC was
  834. * already running on a kernel stack...
  835. */
  836. if (tcstatus & ST0_CU0) {
  837. /* Note that this "- 1" is pointer arithmetic */
  838. kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1;
  839. } else {
  840. kstack = ((struct pt_regs *)kernelsp[cpu]) - 1;
  841. }
  842. kstack->cp0_epc = (long)tcrestart;
  843. /* Save TCStatus */
  844. kstack->cp0_tcstatus = tcstatus;
  845. /* Pass token of operation to be performed kernel stack pad area */
  846. kstack->pad0[4] = (unsigned long)pipi;
  847. /* Pass address of function to be called likewise */
  848. kstack->pad0[5] = (unsigned long)&ipi_decode;
  849. /* Set interrupt exempt and kernel mode */
  850. tcstatus |= TCSTATUS_IXMT;
  851. tcstatus &= ~TCSTATUS_TKSU;
  852. write_tc_c0_tcstatus(tcstatus);
  853. ehb();
  854. /* Set TC Restart address to be SMTC IPI vector */
  855. write_tc_c0_tcrestart(__smtc_ipi_vector);
  856. }
  857. static void ipi_resched_interrupt(void)
  858. {
  859. scheduler_ipi();
  860. }
  861. static void ipi_call_interrupt(void)
  862. {
  863. /* Invoke generic function invocation code in smp.c */
  864. smp_call_function_interrupt();
  865. }
  866. DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device);
  867. static void __irq_entry smtc_clock_tick_interrupt(void)
  868. {
  869. unsigned int cpu = smp_processor_id();
  870. struct clock_event_device *cd;
  871. int irq = MIPS_CPU_IRQ_BASE + 1;
  872. irq_enter();
  873. kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
  874. cd = &per_cpu(mips_clockevent_device, cpu);
  875. cd->event_handler(cd);
  876. irq_exit();
  877. }
  878. void ipi_decode(struct smtc_ipi *pipi)
  879. {
  880. void *arg_copy = pipi->arg;
  881. int type_copy = pipi->type;
  882. smtc_ipi_nq(&freeIPIq, pipi);
  883. switch (type_copy) {
  884. case SMTC_CLOCK_TICK:
  885. smtc_clock_tick_interrupt();
  886. break;
  887. case LINUX_SMP_IPI:
  888. switch ((int)arg_copy) {
  889. case SMP_RESCHEDULE_YOURSELF:
  890. ipi_resched_interrupt();
  891. break;
  892. case SMP_CALL_FUNCTION:
  893. ipi_call_interrupt();
  894. break;
  895. default:
  896. printk("Impossible SMTC IPI Argument %p\n", arg_copy);
  897. break;
  898. }
  899. break;
  900. #ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
  901. case IRQ_AFFINITY_IPI:
  902. /*
  903. * Accept a "forwarded" interrupt that was initially
  904. * taken by a TC who doesn't have affinity for the IRQ.
  905. */
  906. do_IRQ_no_affinity((int)arg_copy);
  907. break;
  908. #endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */
  909. default:
  910. printk("Impossible SMTC IPI Type 0x%x\n", type_copy);
  911. break;
  912. }
  913. }
  914. /*
  915. * Similar to smtc_ipi_replay(), but invoked from context restore,
  916. * so it reuses the current exception frame rather than set up a
  917. * new one with self_ipi.
  918. */
  919. void deferred_smtc_ipi(void)
  920. {
  921. int cpu = smp_processor_id();
  922. /*
  923. * Test is not atomic, but much faster than a dequeue,
  924. * and the vast majority of invocations will have a null queue.
  925. * If irq_disabled when this was called, then any IPIs queued
  926. * after we test last will be taken on the next irq_enable/restore.
  927. * If interrupts were enabled, then any IPIs added after the
  928. * last test will be taken directly.
  929. */
  930. while (IPIQ[cpu].head != NULL) {
  931. struct smtc_ipi_q *q = &IPIQ[cpu];
  932. struct smtc_ipi *pipi;
  933. unsigned long flags;
  934. /*
  935. * It may be possible we'll come in with interrupts
  936. * already enabled.
  937. */
  938. local_irq_save(flags);
  939. spin_lock(&q->lock);
  940. pipi = __smtc_ipi_dq(q);
  941. spin_unlock(&q->lock);
  942. if (pipi != NULL) {
  943. if (pipi->type == LINUX_SMP_IPI &&
  944. (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
  945. IPIQ[cpu].resched_flag = 0;
  946. ipi_decode(pipi);
  947. }
  948. /*
  949. * The use of the __raw_local restore isn't
  950. * as obviously necessary here as in smtc_ipi_replay(),
  951. * but it's more efficient, given that we're already
  952. * running down the IPI queue.
  953. */
  954. __arch_local_irq_restore(flags);
  955. }
  956. }
  957. /*
  958. * Cross-VPE interrupts in the SMTC prototype use "software interrupts"
  959. * set via cross-VPE MTTR manipulation of the Cause register. It would be
  960. * in some regards preferable to have external logic for "doorbell" hardware
  961. * interrupts.
  962. */
  963. static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ;
  964. static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
  965. {
  966. int my_vpe = cpu_data[smp_processor_id()].vpe_id;
  967. int my_tc = cpu_data[smp_processor_id()].tc_id;
  968. int cpu;
  969. struct smtc_ipi *pipi;
  970. unsigned long tcstatus;
  971. int sent;
  972. unsigned long flags;
  973. unsigned int mtflags;
  974. unsigned int vpflags;
  975. /*
  976. * So long as cross-VPE interrupts are done via
  977. * MFTR/MTTR read-modify-writes of Cause, we need
  978. * to stop other VPEs whenever the local VPE does
  979. * anything similar.
  980. */
  981. local_irq_save(flags);
  982. vpflags = dvpe();
  983. clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ);
  984. set_c0_status(0x100 << MIPS_CPU_IPI_IRQ);
  985. irq_enable_hazard();
  986. evpe(vpflags);
  987. local_irq_restore(flags);
  988. /*
  989. * Cross-VPE Interrupt handler: Try to directly deliver IPIs
  990. * queued for TCs on this VPE other than the current one.
  991. * Return-from-interrupt should cause us to drain the queue
  992. * for the current TC, so we ought not to have to do it explicitly here.
  993. */
  994. for_each_online_cpu(cpu) {
  995. if (cpu_data[cpu].vpe_id != my_vpe)
  996. continue;
  997. pipi = smtc_ipi_dq(&IPIQ[cpu]);
  998. if (pipi != NULL) {
  999. if (cpu_data[cpu].tc_id != my_tc) {
  1000. sent = 0;
  1001. LOCK_MT_PRA();
  1002. settc(cpu_data[cpu].tc_id);
  1003. write_tc_c0_tchalt(TCHALT_H);
  1004. mips_ihb();
  1005. tcstatus = read_tc_c0_tcstatus();
  1006. if ((tcstatus & TCSTATUS_IXMT) == 0) {
  1007. post_direct_ipi(cpu, pipi);
  1008. sent = 1;
  1009. }
  1010. write_tc_c0_tchalt(0);
  1011. UNLOCK_MT_PRA();
  1012. if (!sent) {
  1013. smtc_ipi_req(&IPIQ[cpu], pipi);
  1014. }
  1015. } else {
  1016. /*
  1017. * ipi_decode() should be called
  1018. * with interrupts off
  1019. */
  1020. local_irq_save(flags);
  1021. if (pipi->type == LINUX_SMP_IPI &&
  1022. (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
  1023. IPIQ[cpu].resched_flag = 0;
  1024. ipi_decode(pipi);
  1025. local_irq_restore(flags);
  1026. }
  1027. }
  1028. }
  1029. return IRQ_HANDLED;
  1030. }
  1031. static void ipi_irq_dispatch(void)
  1032. {
  1033. do_IRQ(cpu_ipi_irq);
  1034. }
  1035. static struct irqaction irq_ipi = {
  1036. .handler = ipi_interrupt,
  1037. .flags = IRQF_PERCPU,
  1038. .name = "SMTC_IPI"
  1039. };
  1040. static void setup_cross_vpe_interrupts(unsigned int nvpe)
  1041. {
  1042. if (nvpe < 1)
  1043. return;
  1044. if (!cpu_has_vint)
  1045. panic("SMTC Kernel requires Vectored Interrupt support");
  1046. set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch);
  1047. setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ));
  1048. irq_set_handler(cpu_ipi_irq, handle_percpu_irq);
  1049. }
  1050. /*
  1051. * SMTC-specific hacks invoked from elsewhere in the kernel.
  1052. */
  1053. /*
  1054. * smtc_ipi_replay is called from raw_local_irq_restore
  1055. */
  1056. void smtc_ipi_replay(void)
  1057. {
  1058. unsigned int cpu = smp_processor_id();
  1059. /*
  1060. * To the extent that we've ever turned interrupts off,
  1061. * we may have accumulated deferred IPIs. This is subtle.
  1062. * we should be OK: If we pick up something and dispatch
  1063. * it here, that's great. If we see nothing, but concurrent
  1064. * with this operation, another TC sends us an IPI, IXMT
  1065. * is clear, and we'll handle it as a real pseudo-interrupt
  1066. * and not a pseudo-pseudo interrupt. The important thing
  1067. * is to do the last check for queued message *after* the
  1068. * re-enabling of interrupts.
  1069. */
  1070. while (IPIQ[cpu].head != NULL) {
  1071. struct smtc_ipi_q *q = &IPIQ[cpu];
  1072. struct smtc_ipi *pipi;
  1073. unsigned long flags;
  1074. /*
  1075. * It's just possible we'll come in with interrupts
  1076. * already enabled.
  1077. */
  1078. local_irq_save(flags);
  1079. spin_lock(&q->lock);
  1080. pipi = __smtc_ipi_dq(q);
  1081. spin_unlock(&q->lock);
  1082. /*
  1083. ** But use a raw restore here to avoid recursion.
  1084. */
  1085. __arch_local_irq_restore(flags);
  1086. if (pipi) {
  1087. self_ipi(pipi);
  1088. smtc_cpu_stats[cpu].selfipis++;
  1089. }
  1090. }
  1091. }
  1092. EXPORT_SYMBOL(smtc_ipi_replay);
  1093. void smtc_idle_loop_hook(void)
  1094. {
  1095. #ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
  1096. int im;
  1097. int flags;
  1098. int mtflags;
  1099. int bit;
  1100. int vpe;
  1101. int tc;
  1102. int hook_ntcs;
  1103. /*
  1104. * printk within DMT-protected regions can deadlock,
  1105. * so buffer diagnostic messages for later output.
  1106. */
  1107. char *pdb_msg;
  1108. char id_ho_db_msg[768]; /* worst-case use should be less than 700 */
  1109. if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */
  1110. if (atomic_add_return(1, &idle_hook_initialized) == 1) {
  1111. int mvpconf0;
  1112. /* Tedious stuff to just do once */
  1113. mvpconf0 = read_c0_mvpconf0();
  1114. hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
  1115. if (hook_ntcs > NR_CPUS)
  1116. hook_ntcs = NR_CPUS;
  1117. for (tc = 0; tc < hook_ntcs; tc++) {
  1118. tcnoprog[tc] = 0;
  1119. clock_hang_reported[tc] = 0;
  1120. }
  1121. for (vpe = 0; vpe < 2; vpe++)
  1122. for (im = 0; im < 8; im++)
  1123. imstuckcount[vpe][im] = 0;
  1124. printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs);
  1125. atomic_set(&idle_hook_initialized, 1000);
  1126. } else {
  1127. /* Someone else is initializing in parallel - let 'em finish */
  1128. while (atomic_read(&idle_hook_initialized) < 1000)
  1129. ;
  1130. }
  1131. }
  1132. /* Have we stupidly left IXMT set somewhere? */
  1133. if (read_c0_tcstatus() & 0x400) {
  1134. write_c0_tcstatus(read_c0_tcstatus() & ~0x400);
  1135. ehb();
  1136. printk("Dangling IXMT in cpu_idle()\n");
  1137. }
  1138. /* Have we stupidly left an IM bit turned off? */
  1139. #define IM_LIMIT 2000
  1140. local_irq_save(flags);
  1141. mtflags = dmt();
  1142. pdb_msg = &id_ho_db_msg[0];
  1143. im = read_c0_status();
  1144. vpe = current_cpu_data.vpe_id;
  1145. for (bit = 0; bit < 8; bit++) {
  1146. /*
  1147. * In current prototype, I/O interrupts
  1148. * are masked for VPE > 0
  1149. */
  1150. if (vpemask[vpe][bit]) {
  1151. if (!(im & (0x100 << bit)))
  1152. imstuckcount[vpe][bit]++;
  1153. else
  1154. imstuckcount[vpe][bit] = 0;
  1155. if (imstuckcount[vpe][bit] > IM_LIMIT) {
  1156. set_c0_status(0x100 << bit);
  1157. ehb();
  1158. imstuckcount[vpe][bit] = 0;
  1159. pdb_msg += sprintf(pdb_msg,
  1160. "Dangling IM %d fixed for VPE %d\n", bit,
  1161. vpe);
  1162. }
  1163. }
  1164. }
  1165. emt(mtflags);
  1166. local_irq_restore(flags);
  1167. if (pdb_msg != &id_ho_db_msg[0])
  1168. printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg);
  1169. #endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
  1170. smtc_ipi_replay();
  1171. }
  1172. void smtc_soft_dump(void)
  1173. {
  1174. int i;
  1175. printk("Counter Interrupts taken per CPU (TC)\n");
  1176. for (i=0; i < NR_CPUS; i++) {
  1177. printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints);
  1178. }
  1179. printk("Self-IPI invocations:\n");
  1180. for (i=0; i < NR_CPUS; i++) {
  1181. printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis);
  1182. }
  1183. smtc_ipi_qdump();
  1184. printk("%d Recoveries of \"stolen\" FPU\n",
  1185. atomic_read(&smtc_fpu_recoveries));
  1186. }
  1187. /*
  1188. * TLB management routines special to SMTC
  1189. */
  1190. void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
  1191. {
  1192. unsigned long flags, mtflags, tcstat, prevhalt, asid;
  1193. int tlb, i;
  1194. /*
  1195. * It would be nice to be able to use a spinlock here,
  1196. * but this is invoked from within TLB flush routines
  1197. * that protect themselves with DVPE, so if a lock is
  1198. * held by another TC, it'll never be freed.
  1199. *
  1200. * DVPE/DMT must not be done with interrupts enabled,
  1201. * so even so most callers will already have disabled
  1202. * them, let's be really careful...
  1203. */
  1204. local_irq_save(flags);
  1205. if (smtc_status & SMTC_TLB_SHARED) {
  1206. mtflags = dvpe();
  1207. tlb = 0;
  1208. } else {
  1209. mtflags = dmt();
  1210. tlb = cpu_data[cpu].vpe_id;
  1211. }
  1212. asid = asid_cache(cpu);
  1213. do {
  1214. if (!((asid += ASID_INC) & ASID_MASK) ) {
  1215. if (cpu_has_vtag_icache)
  1216. flush_icache_all();
  1217. /* Traverse all online CPUs (hack requires contiguous range) */
  1218. for_each_online_cpu(i) {
  1219. /*
  1220. * We don't need to worry about our own CPU, nor those of
  1221. * CPUs who don't share our TLB.
  1222. */
  1223. if ((i != smp_processor_id()) &&
  1224. ((smtc_status & SMTC_TLB_SHARED) ||
  1225. (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) {
  1226. settc(cpu_data[i].tc_id);
  1227. prevhalt = read_tc_c0_tchalt() & TCHALT_H;
  1228. if (!prevhalt) {
  1229. write_tc_c0_tchalt(TCHALT_H);
  1230. mips_ihb();
  1231. }
  1232. tcstat = read_tc_c0_tcstatus();
  1233. smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i);
  1234. if (!prevhalt)
  1235. write_tc_c0_tchalt(0);
  1236. }
  1237. }
  1238. if (!asid) /* fix version if needed */
  1239. asid = ASID_FIRST_VERSION;
  1240. local_flush_tlb_all(); /* start new asid cycle */
  1241. }
  1242. } while (smtc_live_asid[tlb][(asid & ASID_MASK)]);
  1243. /*
  1244. * SMTC shares the TLB within VPEs and possibly across all VPEs.
  1245. */
  1246. for_each_online_cpu(i) {
  1247. if ((smtc_status & SMTC_TLB_SHARED) ||
  1248. (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))
  1249. cpu_context(i, mm) = asid_cache(i) = asid;
  1250. }
  1251. if (smtc_status & SMTC_TLB_SHARED)
  1252. evpe(mtflags);
  1253. else
  1254. emt(mtflags);
  1255. local_irq_restore(flags);
  1256. }
  1257. /*
  1258. * Invoked from macros defined in mmu_context.h
  1259. * which must already have disabled interrupts
  1260. * and done a DVPE or DMT as appropriate.
  1261. */
  1262. void smtc_flush_tlb_asid(unsigned long asid)
  1263. {
  1264. int entry;
  1265. unsigned long ehi;
  1266. entry = read_c0_wired();
  1267. /* Traverse all non-wired entries */
  1268. while (entry < current_cpu_data.tlbsize) {
  1269. write_c0_index(entry);
  1270. ehb();
  1271. tlb_read();
  1272. ehb();
  1273. ehi = read_c0_entryhi();
  1274. if ((ehi & ASID_MASK) == asid) {
  1275. /*
  1276. * Invalidate only entries with specified ASID,
  1277. * makiing sure all entries differ.
  1278. */
  1279. write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
  1280. write_c0_entrylo0(0);
  1281. write_c0_entrylo1(0);
  1282. mtc0_tlbw_hazard();
  1283. tlb_write_indexed();
  1284. }
  1285. entry++;
  1286. }
  1287. write_c0_index(PARKED_INDEX);
  1288. tlbw_use_hazard();
  1289. }
  1290. /*
  1291. * Support for single-threading cache flush operations.
  1292. */
  1293. static int halt_state_save[NR_CPUS];
  1294. /*
  1295. * To really, really be sure that nothing is being done
  1296. * by other TCs, halt them all. This code assumes that
  1297. * a DVPE has already been done, so while their Halted
  1298. * state is theoretically architecturally unstable, in
  1299. * practice, it's not going to change while we're looking
  1300. * at it.
  1301. */
  1302. void smtc_cflush_lockdown(void)
  1303. {
  1304. int cpu;
  1305. for_each_online_cpu(cpu) {
  1306. if (cpu != smp_processor_id()) {
  1307. settc(cpu_data[cpu].tc_id);
  1308. halt_state_save[cpu] = read_tc_c0_tchalt();
  1309. write_tc_c0_tchalt(TCHALT_H);
  1310. }
  1311. }
  1312. mips_ihb();
  1313. }
  1314. /* It would be cheating to change the cpu_online states during a flush! */
  1315. void smtc_cflush_release(void)
  1316. {
  1317. int cpu;
  1318. /*
  1319. * Start with a hazard barrier to ensure
  1320. * that all CACHE ops have played through.
  1321. */
  1322. mips_ihb();
  1323. for_each_online_cpu(cpu) {
  1324. if (cpu != smp_processor_id()) {
  1325. settc(cpu_data[cpu].tc_id);
  1326. write_tc_c0_tchalt(halt_state_save[cpu]);
  1327. }
  1328. }
  1329. mips_ihb();
  1330. }