entry-armv.S 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140
  1. /*
  2. * linux/arch/arm/kernel/entry-armv.S
  3. *
  4. * Copyright (C) 1996,1997,1998 Russell King.
  5. * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
  6. * nommu support by Hyok S. Choi (hyok.choi@samsung.com)
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License version 2 as
  10. * published by the Free Software Foundation.
  11. *
  12. * Low-level vector interface routines
  13. *
  14. * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction that causes
  15. * it to save wrong values... Be aware!
  16. */
  17. #include <asm/memory.h>
  18. #include <asm/glue.h>
  19. #include <asm/vfpmacros.h>
  20. #include <asm/arch/entry-macro.S>
  21. #include <asm/thread_notify.h>
  22. #include "entry-header.S"
  23. /*
  24. * Interrupt handling. Preserves r7, r8, r9
  25. */
  26. .macro irq_handler
  27. get_irqnr_preamble r5, lr
  28. 1: get_irqnr_and_base r0, r6, r5, lr
  29. movne r1, sp
  30. @
  31. @ routine called with r0 = irq number, r1 = struct pt_regs *
  32. @
  33. adrne lr, 1b
  34. bne asm_do_IRQ
  35. #ifdef CONFIG_SMP
  36. /*
  37. * XXX
  38. *
  39. * this macro assumes that irqstat (r6) and base (r5) are
  40. * preserved from get_irqnr_and_base above
  41. */
  42. test_for_ipi r0, r6, r5, lr
  43. movne r0, sp
  44. adrne lr, 1b
  45. bne do_IPI
  46. #ifdef CONFIG_LOCAL_TIMERS
  47. test_for_ltirq r0, r6, r5, lr
  48. movne r0, sp
  49. adrne lr, 1b
  50. bne do_local_timer
  51. #endif
  52. #endif
  53. .endm
  54. /*
  55. * Invalid mode handlers
  56. */
  57. .macro inv_entry, reason
  58. sub sp, sp, #S_FRAME_SIZE
  59. stmib sp, {r1 - lr}
  60. mov r1, #\reason
  61. .endm
  62. __pabt_invalid:
  63. inv_entry BAD_PREFETCH
  64. b common_invalid
  65. __dabt_invalid:
  66. inv_entry BAD_DATA
  67. b common_invalid
  68. __irq_invalid:
  69. inv_entry BAD_IRQ
  70. b common_invalid
  71. __und_invalid:
  72. inv_entry BAD_UNDEFINSTR
  73. @
  74. @ XXX fall through to common_invalid
  75. @
  76. @
  77. @ common_invalid - generic code for failed exception (re-entrant version of handlers)
  78. @
  79. common_invalid:
  80. zero_fp
  81. ldmia r0, {r4 - r6}
  82. add r0, sp, #S_PC @ here for interlock avoidance
  83. mov r7, #-1 @ "" "" "" ""
  84. str r4, [sp] @ save preserved r0
  85. stmia r0, {r5 - r7} @ lr_<exception>,
  86. @ cpsr_<exception>, "old_r0"
  87. mov r0, sp
  88. b bad_mode
  89. /*
  90. * SVC mode handlers
  91. */
  92. #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
  93. #define SPFIX(code...) code
  94. #else
  95. #define SPFIX(code...)
  96. #endif
  97. .macro svc_entry
  98. sub sp, sp, #S_FRAME_SIZE
  99. SPFIX( tst sp, #4 )
  100. SPFIX( bicne sp, sp, #4 )
  101. stmib sp, {r1 - r12}
  102. ldmia r0, {r1 - r3}
  103. add r5, sp, #S_SP @ here for interlock avoidance
  104. mov r4, #-1 @ "" "" "" ""
  105. add r0, sp, #S_FRAME_SIZE @ "" "" "" ""
  106. SPFIX( addne r0, r0, #4 )
  107. str r1, [sp] @ save the "real" r0 copied
  108. @ from the exception stack
  109. mov r1, lr
  110. @
  111. @ We are now ready to fill in the remaining blanks on the stack:
  112. @
  113. @ r0 - sp_svc
  114. @ r1 - lr_svc
  115. @ r2 - lr_<exception>, already fixed up for correct return/restart
  116. @ r3 - spsr_<exception>
  117. @ r4 - orig_r0 (see pt_regs definition in ptrace.h)
  118. @
  119. stmia r5, {r0 - r4}
  120. .endm
  121. .align 5
  122. __dabt_svc:
  123. svc_entry
  124. @
  125. @ get ready to re-enable interrupts if appropriate
  126. @
  127. mrs r9, cpsr
  128. tst r3, #PSR_I_BIT
  129. biceq r9, r9, #PSR_I_BIT
  130. @
  131. @ Call the processor-specific abort handler:
  132. @
  133. @ r2 - aborted context pc
  134. @ r3 - aborted context cpsr
  135. @
  136. @ The abort handler must return the aborted address in r0, and
  137. @ the fault status register in r1. r9 must be preserved.
  138. @
  139. #ifdef MULTI_ABORT
  140. ldr r4, .LCprocfns
  141. mov lr, pc
  142. ldr pc, [r4]
  143. #else
  144. bl CPU_ABORT_HANDLER
  145. #endif
  146. @
  147. @ set desired IRQ state, then call main handler
  148. @
  149. msr cpsr_c, r9
  150. mov r2, sp
  151. bl do_DataAbort
  152. @
  153. @ IRQs off again before pulling preserved data off the stack
  154. @
  155. disable_irq
  156. @
  157. @ restore SPSR and restart the instruction
  158. @
  159. ldr r0, [sp, #S_PSR]
  160. msr spsr_cxsf, r0
  161. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  162. .align 5
  163. __irq_svc:
  164. svc_entry
  165. #ifdef CONFIG_TRACE_IRQFLAGS
  166. bl trace_hardirqs_off
  167. #endif
  168. #ifdef CONFIG_PREEMPT
  169. get_thread_info tsk
  170. ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
  171. add r7, r8, #1 @ increment it
  172. str r7, [tsk, #TI_PREEMPT]
  173. #endif
  174. irq_handler
  175. #ifdef CONFIG_PREEMPT
  176. ldr r0, [tsk, #TI_FLAGS] @ get flags
  177. tst r0, #_TIF_NEED_RESCHED
  178. blne svc_preempt
  179. preempt_return:
  180. ldr r0, [tsk, #TI_PREEMPT] @ read preempt value
  181. str r8, [tsk, #TI_PREEMPT] @ restore preempt count
  182. teq r0, r7
  183. strne r0, [r0, -r0] @ bug()
  184. #endif
  185. ldr r0, [sp, #S_PSR] @ irqs are already disabled
  186. msr spsr_cxsf, r0
  187. #ifdef CONFIG_TRACE_IRQFLAGS
  188. tst r0, #PSR_I_BIT
  189. bleq trace_hardirqs_on
  190. #endif
  191. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  192. .ltorg
  193. #ifdef CONFIG_PREEMPT
  194. svc_preempt:
  195. teq r8, #0 @ was preempt count = 0
  196. ldreq r6, .LCirq_stat
  197. movne pc, lr @ no
  198. ldr r0, [r6, #4] @ local_irq_count
  199. ldr r1, [r6, #8] @ local_bh_count
  200. adds r0, r0, r1
  201. movne pc, lr
  202. mov r7, #0 @ preempt_schedule_irq
  203. str r7, [tsk, #TI_PREEMPT] @ expects preempt_count == 0
  204. 1: bl preempt_schedule_irq @ irq en/disable is done inside
  205. ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS
  206. tst r0, #_TIF_NEED_RESCHED
  207. beq preempt_return @ go again
  208. b 1b
  209. #endif
  210. .align 5
  211. __und_svc:
  212. svc_entry
  213. @
  214. @ call emulation code, which returns using r9 if it has emulated
  215. @ the instruction, or the more conventional lr if we are to treat
  216. @ this as a real undefined instruction
  217. @
  218. @ r0 - instruction
  219. @
  220. ldr r0, [r2, #-4]
  221. adr r9, 1f
  222. bl call_fpe
  223. mov r0, sp @ struct pt_regs *regs
  224. bl do_undefinstr
  225. @
  226. @ IRQs off again before pulling preserved data off the stack
  227. @
  228. 1: disable_irq
  229. @
  230. @ restore SPSR and restart the instruction
  231. @
  232. ldr lr, [sp, #S_PSR] @ Get SVC cpsr
  233. msr spsr_cxsf, lr
  234. ldmia sp, {r0 - pc}^ @ Restore SVC registers
  235. .align 5
  236. __pabt_svc:
  237. svc_entry
  238. @
  239. @ re-enable interrupts if appropriate
  240. @
  241. mrs r9, cpsr
  242. tst r3, #PSR_I_BIT
  243. biceq r9, r9, #PSR_I_BIT
  244. msr cpsr_c, r9
  245. @
  246. @ set args, then call main handler
  247. @
  248. @ r0 - address of faulting instruction
  249. @ r1 - pointer to registers on stack
  250. @
  251. mov r0, r2 @ address (pc)
  252. mov r1, sp @ regs
  253. bl do_PrefetchAbort @ call abort handler
  254. @
  255. @ IRQs off again before pulling preserved data off the stack
  256. @
  257. disable_irq
  258. @
  259. @ restore SPSR and restart the instruction
  260. @
  261. ldr r0, [sp, #S_PSR]
  262. msr spsr_cxsf, r0
  263. ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
  264. .align 5
  265. .LCcralign:
  266. .word cr_alignment
  267. #ifdef MULTI_ABORT
  268. .LCprocfns:
  269. .word processor
  270. #endif
  271. .LCfp:
  272. .word fp_enter
  273. #ifdef CONFIG_PREEMPT
  274. .LCirq_stat:
  275. .word irq_stat
  276. #endif
  277. /*
  278. * User mode handlers
  279. *
  280. * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE
  281. */
  282. #if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7)
  283. #error "sizeof(struct pt_regs) must be a multiple of 8"
  284. #endif
  285. .macro usr_entry
  286. sub sp, sp, #S_FRAME_SIZE
  287. stmib sp, {r1 - r12}
  288. ldmia r0, {r1 - r3}
  289. add r0, sp, #S_PC @ here for interlock avoidance
  290. mov r4, #-1 @ "" "" "" ""
  291. str r1, [sp] @ save the "real" r0 copied
  292. @ from the exception stack
  293. @
  294. @ We are now ready to fill in the remaining blanks on the stack:
  295. @
  296. @ r2 - lr_<exception>, already fixed up for correct return/restart
  297. @ r3 - spsr_<exception>
  298. @ r4 - orig_r0 (see pt_regs definition in ptrace.h)
  299. @
  300. @ Also, separately save sp_usr and lr_usr
  301. @
  302. stmia r0, {r2 - r4}
  303. stmdb r0, {sp, lr}^
  304. @
  305. @ Enable the alignment trap while in kernel mode
  306. @
  307. alignment_trap r0
  308. @
  309. @ Clear FP to mark the first stack frame
  310. @
  311. zero_fp
  312. .endm
  313. .macro kuser_cmpxchg_check
  314. #if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
  315. #ifndef CONFIG_MMU
  316. #warning "NPTL on non MMU needs fixing"
  317. #else
  318. @ Make sure our user space atomic helper is restarted
  319. @ if it was interrupted in a critical region. Here we
  320. @ perform a quick test inline since it should be false
  321. @ 99.9999% of the time. The rest is done out of line.
  322. cmp r2, #TASK_SIZE
  323. blhs kuser_cmpxchg_fixup
  324. #endif
  325. #endif
  326. .endm
  327. .align 5
  328. __dabt_usr:
  329. usr_entry
  330. kuser_cmpxchg_check
  331. @
  332. @ Call the processor-specific abort handler:
  333. @
  334. @ r2 - aborted context pc
  335. @ r3 - aborted context cpsr
  336. @
  337. @ The abort handler must return the aborted address in r0, and
  338. @ the fault status register in r1.
  339. @
  340. #ifdef MULTI_ABORT
  341. ldr r4, .LCprocfns
  342. mov lr, pc
  343. ldr pc, [r4]
  344. #else
  345. bl CPU_ABORT_HANDLER
  346. #endif
  347. @
  348. @ IRQs on, then call the main handler
  349. @
  350. enable_irq
  351. mov r2, sp
  352. adr lr, ret_from_exception
  353. b do_DataAbort
  354. .align 5
  355. __irq_usr:
  356. usr_entry
  357. kuser_cmpxchg_check
  358. #ifdef CONFIG_TRACE_IRQFLAGS
  359. bl trace_hardirqs_off
  360. #endif
  361. get_thread_info tsk
  362. #ifdef CONFIG_PREEMPT
  363. ldr r8, [tsk, #TI_PREEMPT] @ get preempt count
  364. add r7, r8, #1 @ increment it
  365. str r7, [tsk, #TI_PREEMPT]
  366. #endif
  367. irq_handler
  368. #ifdef CONFIG_PREEMPT
  369. ldr r0, [tsk, #TI_PREEMPT]
  370. str r8, [tsk, #TI_PREEMPT]
  371. teq r0, r7
  372. strne r0, [r0, -r0]
  373. #endif
  374. #ifdef CONFIG_TRACE_IRQFLAGS
  375. bl trace_hardirqs_on
  376. #endif
  377. mov why, #0
  378. b ret_to_user
  379. .ltorg
  380. .align 5
  381. __und_usr:
  382. usr_entry
  383. tst r3, #PSR_T_BIT @ Thumb mode?
  384. bne __und_usr_unknown @ ignore FP
  385. sub r4, r2, #4
  386. @
  387. @ fall through to the emulation code, which returns using r9 if
  388. @ it has emulated the instruction, or the more conventional lr
  389. @ if we are to treat this as a real undefined instruction
  390. @
  391. @ r0 - instruction
  392. @
  393. adr r9, ret_from_exception
  394. adr lr, __und_usr_unknown
  395. 1: ldrt r0, [r4]
  396. @
  397. @ fallthrough to call_fpe
  398. @
  399. /*
  400. * The out of line fixup for the ldrt above.
  401. */
  402. .section .fixup, "ax"
  403. 2: mov pc, r9
  404. .previous
  405. .section __ex_table,"a"
  406. .long 1b, 2b
  407. .previous
  408. /*
  409. * Check whether the instruction is a co-processor instruction.
  410. * If yes, we need to call the relevant co-processor handler.
  411. *
  412. * Note that we don't do a full check here for the co-processor
  413. * instructions; all instructions with bit 27 set are well
  414. * defined. The only instructions that should fault are the
  415. * co-processor instructions. However, we have to watch out
  416. * for the ARM6/ARM7 SWI bug.
  417. *
  418. * NEON is a special case that has to be handled here. Not all
  419. * NEON instructions are co-processor instructions, so we have
  420. * to make a special case of checking for them. Plus, there's
  421. * five groups of them, so we have a table of mask/opcode pairs
  422. * to check against, and if any match then we branch off into the
  423. * NEON handler code.
  424. *
  425. * Emulators may wish to make use of the following registers:
  426. * r0 = instruction opcode.
  427. * r2 = PC+4
  428. * r9 = normal "successful" return address
  429. * r10 = this threads thread_info structure.
  430. * lr = unrecognised instruction return address
  431. */
  432. call_fpe:
  433. #ifdef CONFIG_NEON
  434. adr r6, .LCneon_opcodes
  435. 2:
  436. ldr r7, [r6], #4 @ mask value
  437. cmp r7, #0 @ end mask?
  438. beq 1f
  439. and r8, r0, r7
  440. ldr r7, [r6], #4 @ opcode bits matching in mask
  441. cmp r8, r7 @ NEON instruction?
  442. bne 2b
  443. get_thread_info r10
  444. mov r7, #1
  445. strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used
  446. strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used
  447. b do_vfp @ let VFP handler handle this
  448. 1:
  449. #endif
  450. tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27
  451. #if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
  452. and r8, r0, #0x0f000000 @ mask out op-code bits
  453. teqne r8, #0x0f000000 @ SWI (ARM6/7 bug)?
  454. #endif
  455. moveq pc, lr
  456. get_thread_info r10 @ get current thread
  457. and r8, r0, #0x00000f00 @ mask out CP number
  458. mov r7, #1
  459. add r6, r10, #TI_USED_CP
  460. strb r7, [r6, r8, lsr #8] @ set appropriate used_cp[]
  461. #ifdef CONFIG_IWMMXT
  462. @ Test if we need to give access to iWMMXt coprocessors
  463. ldr r5, [r10, #TI_FLAGS]
  464. rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only
  465. movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1)
  466. bcs iwmmxt_task_enable
  467. #endif
  468. add pc, pc, r8, lsr #6
  469. mov r0, r0
  470. mov pc, lr @ CP#0
  471. b do_fpe @ CP#1 (FPE)
  472. b do_fpe @ CP#2 (FPE)
  473. mov pc, lr @ CP#3
  474. #ifdef CONFIG_CRUNCH
  475. b crunch_task_enable @ CP#4 (MaverickCrunch)
  476. b crunch_task_enable @ CP#5 (MaverickCrunch)
  477. b crunch_task_enable @ CP#6 (MaverickCrunch)
  478. #else
  479. mov pc, lr @ CP#4
  480. mov pc, lr @ CP#5
  481. mov pc, lr @ CP#6
  482. #endif
  483. mov pc, lr @ CP#7
  484. mov pc, lr @ CP#8
  485. mov pc, lr @ CP#9
  486. #ifdef CONFIG_VFP
  487. b do_vfp @ CP#10 (VFP)
  488. b do_vfp @ CP#11 (VFP)
  489. #else
  490. mov pc, lr @ CP#10 (VFP)
  491. mov pc, lr @ CP#11 (VFP)
  492. #endif
  493. mov pc, lr @ CP#12
  494. mov pc, lr @ CP#13
  495. mov pc, lr @ CP#14 (Debug)
  496. mov pc, lr @ CP#15 (Control)
  497. #ifdef CONFIG_NEON
  498. .align 6
  499. .LCneon_opcodes:
  500. .word 0xfe000000 @ mask
  501. .word 0xf2000000 @ opcode
  502. .word 0xff100000 @ mask
  503. .word 0xf4000000 @ opcode
  504. .word 0x00000000 @ mask
  505. .word 0x00000000 @ opcode
  506. #endif
  507. do_fpe:
  508. enable_irq
  509. ldr r4, .LCfp
  510. add r10, r10, #TI_FPSTATE @ r10 = workspace
  511. ldr pc, [r4] @ Call FP module USR entry point
  512. /*
  513. * The FP module is called with these registers set:
  514. * r0 = instruction
  515. * r2 = PC+4
  516. * r9 = normal "successful" return address
  517. * r10 = FP workspace
  518. * lr = unrecognised FP instruction return address
  519. */
  520. .data
  521. ENTRY(fp_enter)
  522. .word no_fp
  523. .text
  524. no_fp: mov pc, lr
  525. __und_usr_unknown:
  526. mov r0, sp
  527. adr lr, ret_from_exception
  528. b do_undefinstr
  529. .align 5
  530. __pabt_usr:
  531. usr_entry
  532. enable_irq @ Enable interrupts
  533. mov r0, r2 @ address (pc)
  534. mov r1, sp @ regs
  535. bl do_PrefetchAbort @ call abort handler
  536. /* fall through */
  537. /*
  538. * This is the return code to user mode for abort handlers
  539. */
  540. ENTRY(ret_from_exception)
  541. get_thread_info tsk
  542. mov why, #0
  543. b ret_to_user
  544. /*
  545. * Register switch for ARMv3 and ARMv4 processors
  546. * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info
  547. * previous and next are guaranteed not to be the same.
  548. */
  549. ENTRY(__switch_to)
  550. add ip, r1, #TI_CPU_SAVE
  551. ldr r3, [r2, #TI_TP_VALUE]
  552. stmia ip!, {r4 - sl, fp, sp, lr} @ Store most regs on stack
  553. #ifdef CONFIG_MMU
  554. ldr r6, [r2, #TI_CPU_DOMAIN]
  555. #endif
  556. #if __LINUX_ARM_ARCH__ >= 6
  557. #ifdef CONFIG_CPU_32v6K
  558. clrex
  559. #else
  560. strex r5, r4, [ip] @ Clear exclusive monitor
  561. #endif
  562. #endif
  563. #if defined(CONFIG_HAS_TLS_REG)
  564. mcr p15, 0, r3, c13, c0, 3 @ set TLS register
  565. #elif !defined(CONFIG_TLS_REG_EMUL)
  566. mov r4, #0xffff0fff
  567. str r3, [r4, #-15] @ TLS val at 0xffff0ff0
  568. #endif
  569. #ifdef CONFIG_MMU
  570. mcr p15, 0, r6, c3, c0, 0 @ Set domain register
  571. #endif
  572. mov r5, r0
  573. add r4, r2, #TI_CPU_SAVE
  574. ldr r0, =thread_notify_head
  575. mov r1, #THREAD_NOTIFY_SWITCH
  576. bl atomic_notifier_call_chain
  577. mov r0, r5
  578. ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously
  579. __INIT
  580. /*
  581. * User helpers.
  582. *
  583. * These are segment of kernel provided user code reachable from user space
  584. * at a fixed address in kernel memory. This is used to provide user space
  585. * with some operations which require kernel help because of unimplemented
  586. * native feature and/or instructions in many ARM CPUs. The idea is for
  587. * this code to be executed directly in user mode for best efficiency but
  588. * which is too intimate with the kernel counter part to be left to user
  589. * libraries. In fact this code might even differ from one CPU to another
  590. * depending on the available instruction set and restrictions like on
  591. * SMP systems. In other words, the kernel reserves the right to change
  592. * this code as needed without warning. Only the entry points and their
  593. * results are guaranteed to be stable.
  594. *
  595. * Each segment is 32-byte aligned and will be moved to the top of the high
  596. * vector page. New segments (if ever needed) must be added in front of
  597. * existing ones. This mechanism should be used only for things that are
  598. * really small and justified, and not be abused freely.
  599. *
  600. * User space is expected to implement those things inline when optimizing
  601. * for a processor that has the necessary native support, but only if such
  602. * resulting binaries are already to be incompatible with earlier ARM
  603. * processors due to the use of unsupported instructions other than what
  604. * is provided here. In other words don't make binaries unable to run on
  605. * earlier processors just for the sake of not using these kernel helpers
  606. * if your compiled code is not going to use the new instructions for other
  607. * purpose.
  608. */
  609. .macro usr_ret, reg
  610. #ifdef CONFIG_ARM_THUMB
  611. bx \reg
  612. #else
  613. mov pc, \reg
  614. #endif
  615. .endm
  616. .align 5
  617. .globl __kuser_helper_start
  618. __kuser_helper_start:
  619. /*
  620. * Reference prototype:
  621. *
  622. * void __kernel_memory_barrier(void)
  623. *
  624. * Input:
  625. *
  626. * lr = return address
  627. *
  628. * Output:
  629. *
  630. * none
  631. *
  632. * Clobbered:
  633. *
  634. * none
  635. *
  636. * Definition and user space usage example:
  637. *
  638. * typedef void (__kernel_dmb_t)(void);
  639. * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0)
  640. *
  641. * Apply any needed memory barrier to preserve consistency with data modified
  642. * manually and __kuser_cmpxchg usage.
  643. *
  644. * This could be used as follows:
  645. *
  646. * #define __kernel_dmb() \
  647. * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \
  648. * : : : "r0", "lr","cc" )
  649. */
  650. __kuser_memory_barrier: @ 0xffff0fa0
  651. #if __LINUX_ARM_ARCH__ >= 6 && defined(CONFIG_SMP)
  652. mcr p15, 0, r0, c7, c10, 5 @ dmb
  653. #endif
  654. usr_ret lr
  655. .align 5
  656. /*
  657. * Reference prototype:
  658. *
  659. * int __kernel_cmpxchg(int oldval, int newval, int *ptr)
  660. *
  661. * Input:
  662. *
  663. * r0 = oldval
  664. * r1 = newval
  665. * r2 = ptr
  666. * lr = return address
  667. *
  668. * Output:
  669. *
  670. * r0 = returned value (zero or non-zero)
  671. * C flag = set if r0 == 0, clear if r0 != 0
  672. *
  673. * Clobbered:
  674. *
  675. * r3, ip, flags
  676. *
  677. * Definition and user space usage example:
  678. *
  679. * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr);
  680. * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0)
  681. *
  682. * Atomically store newval in *ptr if *ptr is equal to oldval for user space.
  683. * Return zero if *ptr was changed or non-zero if no exchange happened.
  684. * The C flag is also set if *ptr was changed to allow for assembly
  685. * optimization in the calling code.
  686. *
  687. * Notes:
  688. *
  689. * - This routine already includes memory barriers as needed.
  690. *
  691. * For example, a user space atomic_add implementation could look like this:
  692. *
  693. * #define atomic_add(ptr, val) \
  694. * ({ register unsigned int *__ptr asm("r2") = (ptr); \
  695. * register unsigned int __result asm("r1"); \
  696. * asm volatile ( \
  697. * "1: @ atomic_add\n\t" \
  698. * "ldr r0, [r2]\n\t" \
  699. * "mov r3, #0xffff0fff\n\t" \
  700. * "add lr, pc, #4\n\t" \
  701. * "add r1, r0, %2\n\t" \
  702. * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \
  703. * "bcc 1b" \
  704. * : "=&r" (__result) \
  705. * : "r" (__ptr), "rIL" (val) \
  706. * : "r0","r3","ip","lr","cc","memory" ); \
  707. * __result; })
  708. */
  709. __kuser_cmpxchg: @ 0xffff0fc0
  710. #if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG)
  711. /*
  712. * Poor you. No fast solution possible...
  713. * The kernel itself must perform the operation.
  714. * A special ghost syscall is used for that (see traps.c).
  715. */
  716. stmfd sp!, {r7, lr}
  717. mov r7, #0xff00 @ 0xfff0 into r7 for EABI
  718. orr r7, r7, #0xf0
  719. swi #0x9ffff0
  720. ldmfd sp!, {r7, pc}
  721. #elif __LINUX_ARM_ARCH__ < 6
  722. #ifdef CONFIG_MMU
  723. /*
  724. * The only thing that can break atomicity in this cmpxchg
  725. * implementation is either an IRQ or a data abort exception
  726. * causing another process/thread to be scheduled in the middle
  727. * of the critical sequence. To prevent this, code is added to
  728. * the IRQ and data abort exception handlers to set the pc back
  729. * to the beginning of the critical section if it is found to be
  730. * within that critical section (see kuser_cmpxchg_fixup).
  731. */
  732. 1: ldr r3, [r2] @ load current val
  733. subs r3, r3, r0 @ compare with oldval
  734. 2: streq r1, [r2] @ store newval if eq
  735. rsbs r0, r3, #0 @ set return val and C flag
  736. usr_ret lr
  737. .text
  738. kuser_cmpxchg_fixup:
  739. @ Called from kuser_cmpxchg_check macro.
  740. @ r2 = address of interrupted insn (must be preserved).
  741. @ sp = saved regs. r7 and r8 are clobbered.
  742. @ 1b = first critical insn, 2b = last critical insn.
  743. @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b.
  744. mov r7, #0xffff0fff
  745. sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
  746. subs r8, r2, r7
  747. rsbcss r8, r8, #(2b - 1b)
  748. strcs r7, [sp, #S_PC]
  749. mov pc, lr
  750. .previous
  751. #else
  752. #warning "NPTL on non MMU needs fixing"
  753. mov r0, #-1
  754. adds r0, r0, #0
  755. usr_ret lr
  756. #endif
  757. #else
  758. #ifdef CONFIG_SMP
  759. mcr p15, 0, r0, c7, c10, 5 @ dmb
  760. #endif
  761. 1: ldrex r3, [r2]
  762. subs r3, r3, r0
  763. strexeq r3, r1, [r2]
  764. teqeq r3, #1
  765. beq 1b
  766. rsbs r0, r3, #0
  767. /* beware -- each __kuser slot must be 8 instructions max */
  768. #ifdef CONFIG_SMP
  769. b __kuser_memory_barrier
  770. #else
  771. usr_ret lr
  772. #endif
  773. #endif
  774. .align 5
  775. /*
  776. * Reference prototype:
  777. *
  778. * int __kernel_get_tls(void)
  779. *
  780. * Input:
  781. *
  782. * lr = return address
  783. *
  784. * Output:
  785. *
  786. * r0 = TLS value
  787. *
  788. * Clobbered:
  789. *
  790. * none
  791. *
  792. * Definition and user space usage example:
  793. *
  794. * typedef int (__kernel_get_tls_t)(void);
  795. * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0)
  796. *
  797. * Get the TLS value as previously set via the __ARM_NR_set_tls syscall.
  798. *
  799. * This could be used as follows:
  800. *
  801. * #define __kernel_get_tls() \
  802. * ({ register unsigned int __val asm("r0"); \
  803. * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \
  804. * : "=r" (__val) : : "lr","cc" ); \
  805. * __val; })
  806. */
  807. __kuser_get_tls: @ 0xffff0fe0
  808. #if !defined(CONFIG_HAS_TLS_REG) && !defined(CONFIG_TLS_REG_EMUL)
  809. ldr r0, [pc, #(16 - 8)] @ TLS stored at 0xffff0ff0
  810. #else
  811. mrc p15, 0, r0, c13, c0, 3 @ read TLS register
  812. #endif
  813. usr_ret lr
  814. .rep 5
  815. .word 0 @ pad up to __kuser_helper_version
  816. .endr
  817. /*
  818. * Reference declaration:
  819. *
  820. * extern unsigned int __kernel_helper_version;
  821. *
  822. * Definition and user space usage example:
  823. *
  824. * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc)
  825. *
  826. * User space may read this to determine the curent number of helpers
  827. * available.
  828. */
  829. __kuser_helper_version: @ 0xffff0ffc
  830. .word ((__kuser_helper_end - __kuser_helper_start) >> 5)
  831. .globl __kuser_helper_end
  832. __kuser_helper_end:
  833. /*
  834. * Vector stubs.
  835. *
  836. * This code is copied to 0xffff0200 so we can use branches in the
  837. * vectors, rather than ldr's. Note that this code must not
  838. * exceed 0x300 bytes.
  839. *
  840. * Common stub entry macro:
  841. * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
  842. *
  843. * SP points to a minimal amount of processor-private memory, the address
  844. * of which is copied into r0 for the mode specific abort handler.
  845. */
  846. .macro vector_stub, name, mode, correction=0
  847. .align 5
  848. vector_\name:
  849. .if \correction
  850. sub lr, lr, #\correction
  851. .endif
  852. @
  853. @ Save r0, lr_<exception> (parent PC) and spsr_<exception>
  854. @ (parent CPSR)
  855. @
  856. stmia sp, {r0, lr} @ save r0, lr
  857. mrs lr, spsr
  858. str lr, [sp, #8] @ save spsr
  859. @
  860. @ Prepare for SVC32 mode. IRQs remain disabled.
  861. @
  862. mrs r0, cpsr
  863. eor r0, r0, #(\mode ^ SVC_MODE)
  864. msr spsr_cxsf, r0
  865. @
  866. @ the branch table must immediately follow this code
  867. @
  868. and lr, lr, #0x0f
  869. mov r0, sp
  870. ldr lr, [pc, lr, lsl #2]
  871. movs pc, lr @ branch to handler in SVC mode
  872. .endm
  873. .globl __stubs_start
  874. __stubs_start:
  875. /*
  876. * Interrupt dispatcher
  877. */
  878. vector_stub irq, IRQ_MODE, 4
  879. .long __irq_usr @ 0 (USR_26 / USR_32)
  880. .long __irq_invalid @ 1 (FIQ_26 / FIQ_32)
  881. .long __irq_invalid @ 2 (IRQ_26 / IRQ_32)
  882. .long __irq_svc @ 3 (SVC_26 / SVC_32)
  883. .long __irq_invalid @ 4
  884. .long __irq_invalid @ 5
  885. .long __irq_invalid @ 6
  886. .long __irq_invalid @ 7
  887. .long __irq_invalid @ 8
  888. .long __irq_invalid @ 9
  889. .long __irq_invalid @ a
  890. .long __irq_invalid @ b
  891. .long __irq_invalid @ c
  892. .long __irq_invalid @ d
  893. .long __irq_invalid @ e
  894. .long __irq_invalid @ f
  895. /*
  896. * Data abort dispatcher
  897. * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
  898. */
  899. vector_stub dabt, ABT_MODE, 8
  900. .long __dabt_usr @ 0 (USR_26 / USR_32)
  901. .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32)
  902. .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32)
  903. .long __dabt_svc @ 3 (SVC_26 / SVC_32)
  904. .long __dabt_invalid @ 4
  905. .long __dabt_invalid @ 5
  906. .long __dabt_invalid @ 6
  907. .long __dabt_invalid @ 7
  908. .long __dabt_invalid @ 8
  909. .long __dabt_invalid @ 9
  910. .long __dabt_invalid @ a
  911. .long __dabt_invalid @ b
  912. .long __dabt_invalid @ c
  913. .long __dabt_invalid @ d
  914. .long __dabt_invalid @ e
  915. .long __dabt_invalid @ f
  916. /*
  917. * Prefetch abort dispatcher
  918. * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
  919. */
  920. vector_stub pabt, ABT_MODE, 4
  921. .long __pabt_usr @ 0 (USR_26 / USR_32)
  922. .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32)
  923. .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32)
  924. .long __pabt_svc @ 3 (SVC_26 / SVC_32)
  925. .long __pabt_invalid @ 4
  926. .long __pabt_invalid @ 5
  927. .long __pabt_invalid @ 6
  928. .long __pabt_invalid @ 7
  929. .long __pabt_invalid @ 8
  930. .long __pabt_invalid @ 9
  931. .long __pabt_invalid @ a
  932. .long __pabt_invalid @ b
  933. .long __pabt_invalid @ c
  934. .long __pabt_invalid @ d
  935. .long __pabt_invalid @ e
  936. .long __pabt_invalid @ f
  937. /*
  938. * Undef instr entry dispatcher
  939. * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
  940. */
  941. vector_stub und, UND_MODE
  942. .long __und_usr @ 0 (USR_26 / USR_32)
  943. .long __und_invalid @ 1 (FIQ_26 / FIQ_32)
  944. .long __und_invalid @ 2 (IRQ_26 / IRQ_32)
  945. .long __und_svc @ 3 (SVC_26 / SVC_32)
  946. .long __und_invalid @ 4
  947. .long __und_invalid @ 5
  948. .long __und_invalid @ 6
  949. .long __und_invalid @ 7
  950. .long __und_invalid @ 8
  951. .long __und_invalid @ 9
  952. .long __und_invalid @ a
  953. .long __und_invalid @ b
  954. .long __und_invalid @ c
  955. .long __und_invalid @ d
  956. .long __und_invalid @ e
  957. .long __und_invalid @ f
  958. .align 5
  959. /*=============================================================================
  960. * Undefined FIQs
  961. *-----------------------------------------------------------------------------
  962. * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC
  963. * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg.
  964. * Basically to switch modes, we *HAVE* to clobber one register... brain
  965. * damage alert! I don't think that we can execute any code in here in any
  966. * other mode than FIQ... Ok you can switch to another mode, but you can't
  967. * get out of that mode without clobbering one register.
  968. */
  969. vector_fiq:
  970. disable_fiq
  971. subs pc, lr, #4
  972. /*=============================================================================
  973. * Address exception handler
  974. *-----------------------------------------------------------------------------
  975. * These aren't too critical.
  976. * (they're not supposed to happen, and won't happen in 32-bit data mode).
  977. */
  978. vector_addrexcptn:
  979. b vector_addrexcptn
  980. /*
  981. * We group all the following data together to optimise
  982. * for CPUs with separate I & D caches.
  983. */
  984. .align 5
  985. .LCvswi:
  986. .word vector_swi
  987. .globl __stubs_end
  988. __stubs_end:
  989. .equ stubs_offset, __vectors_start + 0x200 - __stubs_start
  990. .globl __vectors_start
  991. __vectors_start:
  992. swi SYS_ERROR0
  993. b vector_und + stubs_offset
  994. ldr pc, .LCvswi + stubs_offset
  995. b vector_pabt + stubs_offset
  996. b vector_dabt + stubs_offset
  997. b vector_addrexcptn + stubs_offset
  998. b vector_irq + stubs_offset
  999. b vector_fiq + stubs_offset
  1000. .globl __vectors_end
  1001. __vectors_end:
  1002. .data
  1003. .globl cr_alignment
  1004. .globl cr_no_alignment
  1005. cr_alignment:
  1006. .space 4
  1007. cr_no_alignment:
  1008. .space 4