Line data Source code
1 : #ifndef _ASM_X86_SYSTEM_H
2 : #define _ASM_X86_SYSTEM_H
3 :
4 : #include <asm/asm.h>
5 : #include <asm/segment.h>
6 : #include <asm/cpufeature.h>
7 : #include <asm/cmpxchg.h>
8 : #include <asm/nops.h>
9 :
10 : #include <linux/kernel.h>
11 : #include <linux/irqflags.h>
12 :
13 : /* entries in ARCH_DLINFO: */
14 : #if defined(CONFIG_IA32_EMULATION) || !defined(CONFIG_X86_64)
15 : # define AT_VECTOR_SIZE_ARCH 2
16 : #else /* else it's non-compat x86-64 */
17 : # define AT_VECTOR_SIZE_ARCH 1
18 : #endif
19 :
20 : struct task_struct; /* one of the stranger aspects of C forward declarations */
21 : struct task_struct *__switch_to(struct task_struct *prev,
22 : struct task_struct *next);
23 : struct tss_struct;
24 : void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
25 : struct tss_struct *tss);
26 : extern void show_regs_common(void);
27 :
28 : #ifdef CONFIG_X86_32
29 :
30 : #ifdef CONFIG_CC_STACKPROTECTOR
31 : #define __switch_canary \
32 : "movl %P[task_canary](%[next]), %%ebx\n\t" \
33 : "movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
34 : #define __switch_canary_oparam \
35 : , [stack_canary] "=m" (per_cpu_var(stack_canary.canary))
36 : #define __switch_canary_iparam \
37 : , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
38 : #else /* CC_STACKPROTECTOR */
39 : #define __switch_canary
40 : #define __switch_canary_oparam
41 : #define __switch_canary_iparam
42 : #endif /* CC_STACKPROTECTOR */
43 :
44 : /*
45 : * Saving eflags is important. It switches not only IOPL between tasks,
46 : * it also protects other tasks from NT leaking through sysenter etc.
47 : */
48 : #define switch_to(prev, next, last) \
49 : do { \
50 : /* \
51 : * Context-switching clobbers all registers, so we clobber \
52 : * them explicitly, via unused output variables. \
53 : * (EAX and EBP is not listed because EBP is saved/restored \
54 : * explicitly for wchan access and EAX is the return value of \
55 : * __switch_to()) \
56 : */ \
57 : unsigned long ebx, ecx, edx, esi, edi; \
58 : \
59 : asm volatile("pushfl\n\t" /* save flags */ \
60 : "pushl %%ebp\n\t" /* save EBP */ \
61 : "movl %%esp,%[prev_sp]\n\t" /* save ESP */ \
62 : "movl %[next_sp],%%esp\n\t" /* restore ESP */ \
63 : "movl $1f,%[prev_ip]\n\t" /* save EIP */ \
64 : "pushl %[next_ip]\n\t" /* restore EIP */ \
65 : __switch_canary \
66 : "jmp __switch_to\n" /* regparm call */ \
67 : "1:\t" \
68 : "popl %%ebp\n\t" /* restore EBP */ \
69 : "popfl\n" /* restore flags */ \
70 : \
71 : /* output parameters */ \
72 : : [prev_sp] "=m" (prev->thread.sp), \
73 : [prev_ip] "=m" (prev->thread.ip), \
74 : "=a" (last), \
75 : \
76 : /* clobbered output registers: */ \
77 : "=b" (ebx), "=c" (ecx), "=d" (edx), \
78 : "=S" (esi), "=D" (edi) \
79 : \
80 : __switch_canary_oparam \
81 : \
82 : /* input parameters: */ \
83 : : [next_sp] "m" (next->thread.sp), \
84 : [next_ip] "m" (next->thread.ip), \
85 : \
86 : /* regparm parameters for __switch_to(): */ \
87 : [prev] "a" (prev), \
88 : [next] "d" (next) \
89 : \
90 : __switch_canary_iparam \
91 : \
92 : : /* reloaded segment registers */ \
93 : "memory"); \
94 : } while (0)
95 :
96 : /*
97 : * disable hlt during certain critical i/o operations
98 : */
99 : #define HAVE_DISABLE_HLT
100 : #else
101 : #define __SAVE(reg, offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
102 : #define __RESTORE(reg, offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"
103 :
104 : /* frame pointer must be last for get_wchan */
105 : #define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
106 : #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"
107 :
108 : #define __EXTRA_CLOBBER \
109 : , "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
110 : "r12", "r13", "r14", "r15"
111 :
112 : #ifdef CONFIG_CC_STACKPROTECTOR
113 : #define __switch_canary \
114 : "movq %P[task_canary](%%rsi),%%r8\n\t" \
115 : "movq %%r8,"__percpu_arg([gs_canary])"\n\t"
116 : #define __switch_canary_oparam \
117 : , [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary))
118 : #define __switch_canary_iparam \
119 : , [task_canary] "i" (offsetof(struct task_struct, stack_canary))
120 : #else /* CC_STACKPROTECTOR */
121 : #define __switch_canary
122 : #define __switch_canary_oparam
123 : #define __switch_canary_iparam
124 : #endif /* CC_STACKPROTECTOR */
125 :
126 : /* Save restore flags to clear handle leaking NT */
127 : #define switch_to(prev, next, last) \
128 : asm volatile(SAVE_CONTEXT \
129 : "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
130 : "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
131 : "call __switch_to\n\t" \
132 : "movq "__percpu_arg([current_task])",%%rsi\n\t" \
133 : __switch_canary \
134 : "movq %P[thread_info](%%rsi),%%r8\n\t" \
135 : "movq %%rax,%%rdi\n\t" \
136 : "testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
137 : "jnz ret_from_fork\n\t" \
138 : RESTORE_CONTEXT \
139 : : "=a" (last) \
140 : __switch_canary_oparam \
141 : : [next] "S" (next), [prev] "D" (prev), \
142 : [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
143 : [ti_flags] "i" (offsetof(struct thread_info, flags)), \
144 : [_tif_fork] "i" (_TIF_FORK), \
145 : [thread_info] "i" (offsetof(struct task_struct, stack)), \
146 : [current_task] "m" (per_cpu_var(current_task)) \
147 : __switch_canary_iparam \
148 : : "memory", "cc" __EXTRA_CLOBBER)
149 : #endif
150 :
151 : #ifdef __KERNEL__
152 :
153 : extern void native_load_gs_index(unsigned);
154 :
155 : /*
156 : * Load a segment. Fall back on loading the zero
157 : * segment if something goes wrong..
158 : */
159 : #define loadsegment(seg, value) \
160 : do { \
161 : unsigned short __val = (value); \
162 : \
163 : asm volatile(" \n" \
164 : "1: movl %k0,%%" #seg " \n" \
165 : \
166 : ".section .fixup,\"ax\" \n" \
167 : "2: xorl %k0,%k0 \n" \
168 : " jmp 1b \n" \
169 : ".previous \n" \
170 : \
171 : _ASM_EXTABLE(1b, 2b) \
172 : \
173 : : "+r" (__val) : : "memory"); \
174 : } while (0)
175 :
176 : /*
177 : * Save a segment register away
178 : */
179 : #define savesegment(seg, value) \
180 : asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
181 :
182 : /*
183 : * x86_32 user gs accessors.
184 : */
185 : #ifdef CONFIG_X86_32
186 : #ifdef CONFIG_X86_32_LAZY_GS
187 : #define get_user_gs(regs) (u16)({unsigned long v; savesegment(gs, v); v;})
188 : #define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v))
189 : #define task_user_gs(tsk) ((tsk)->thread.gs)
190 : #define lazy_save_gs(v) savesegment(gs, (v))
191 : #define lazy_load_gs(v) loadsegment(gs, (v))
192 : #else /* X86_32_LAZY_GS */
193 : #define get_user_gs(regs) (u16)((regs)->gs)
194 : #define set_user_gs(regs, v) do { (regs)->gs = (v); } while (0)
195 : #define task_user_gs(tsk) (task_pt_regs(tsk)->gs)
196 : #define lazy_save_gs(v) do { } while (0)
197 : #define lazy_load_gs(v) do { } while (0)
198 : #endif /* X86_32_LAZY_GS */
199 : #endif /* X86_32 */
200 :
201 : static inline unsigned long get_limit(unsigned long segment)
202 : {
203 : unsigned long __limit;
204 : asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
205 : return __limit + 1;
206 : }
207 :
208 : static inline void native_clts(void)
209 : {
210 : asm volatile("clts");
211 : }
212 :
213 : /*
214 : * Volatile isn't enough to prevent the compiler from reordering the
215 : * read/write functions for the control registers and messing everything up.
216 : * A memory clobber would solve the problem, but would prevent reordering of
217 : * all loads stores around it, which can hurt performance. Solution is to
218 : * use a variable and mimic reads and writes to it to enforce serialization
219 : */
220 : static unsigned long __force_order;
221 :
222 : static inline unsigned long native_read_cr0(void)
223 : {
224 : unsigned long val;
225 : asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
226 : return val;
227 : }
228 :
229 : static inline void native_write_cr0(unsigned long val)
230 : {
231 : asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
232 : }
233 :
234 : static inline unsigned long native_read_cr2(void)
235 : {
236 : unsigned long val;
237 : asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
238 : return val;
239 : }
240 :
241 : static inline void native_write_cr2(unsigned long val)
242 : {
243 : asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
244 : }
245 :
246 : static inline unsigned long native_read_cr3(void)
247 : {
248 : unsigned long val;
249 : asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
250 : return val;
251 : }
252 :
253 : static inline void native_write_cr3(unsigned long val)
254 : {
255 : asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
256 : }
257 :
258 : static inline unsigned long native_read_cr4(void)
259 : {
260 : unsigned long val;
261 : asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
262 : return val;
263 : }
264 :
265 : static inline unsigned long native_read_cr4_safe(void)
266 : {
267 : unsigned long val;
268 : /* This could fault if %cr4 does not exist. In x86_64, a cr4 always
269 : * exists, so it will never fail. */
270 : #ifdef CONFIG_X86_32
271 : asm volatile("1: mov %%cr4, %0\n"
272 : "2:\n"
273 : _ASM_EXTABLE(1b, 2b)
274 : : "=r" (val), "=m" (__force_order) : "0" (0));
275 : #else
276 : val = native_read_cr4();
277 : #endif
278 : return val;
279 : }
280 :
281 : static inline void native_write_cr4(unsigned long val)
282 : {
283 : asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
284 : }
285 :
286 : #ifdef CONFIG_X86_64
287 : static inline unsigned long native_read_cr8(void)
288 : {
289 : unsigned long cr8;
290 : asm volatile("movq %%cr8,%0" : "=r" (cr8));
291 : return cr8;
292 : }
293 :
294 : static inline void native_write_cr8(unsigned long val)
295 : {
296 : asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
297 : }
298 : #endif
299 :
300 : static inline void native_wbinvd(void)
301 : {
302 : asm volatile("wbinvd": : :"memory");
303 : }
304 :
305 : #ifdef CONFIG_PARAVIRT
306 : #include <asm/paravirt.h>
307 : #else
308 : #define read_cr0() (native_read_cr0())
309 : #define write_cr0(x) (native_write_cr0(x))
310 : #define read_cr2() (native_read_cr2())
311 : #define write_cr2(x) (native_write_cr2(x))
312 : #define read_cr3() (native_read_cr3())
313 : #define write_cr3(x) (native_write_cr3(x))
314 : #define read_cr4() (native_read_cr4())
315 : #define read_cr4_safe() (native_read_cr4_safe())
316 : #define write_cr4(x) (native_write_cr4(x))
317 : #define wbinvd() (native_wbinvd())
318 : #ifdef CONFIG_X86_64
319 : #define read_cr8() (native_read_cr8())
320 : #define write_cr8(x) (native_write_cr8(x))
321 : #define load_gs_index native_load_gs_index
322 : #endif
323 :
324 : /* Clear the 'TS' bit */
325 : #define clts() (native_clts())
326 :
327 : #endif/* CONFIG_PARAVIRT */
328 :
329 : #define stts() write_cr0(read_cr0() | X86_CR0_TS)
330 :
331 : #endif /* __KERNEL__ */
332 :
333 : static inline void clflush(volatile void *__p)
334 : {
335 : asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
336 : }
337 :
338 : #define nop() asm volatile ("nop")
339 :
340 : void disable_hlt(void);
341 : void enable_hlt(void);
342 :
343 : void cpu_idle_wait(void);
344 :
345 : extern unsigned long arch_align_stack(unsigned long sp);
346 : extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
347 :
348 : void default_idle(void);
349 :
350 : void stop_this_cpu(void *dummy);
351 :
352 : /*
353 : * Force strict CPU ordering.
354 : * And yes, this is required on UP too when we're talking
355 : * to devices.
356 : */
357 : #ifdef CONFIG_X86_32
358 : /*
359 : * Some non-Intel clones support out of order store. wmb() ceases to be a
360 : * nop for these.
361 : */
362 : #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
363 : #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
364 : #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
365 : #else
366 : #define mb() asm volatile("mfence":::"memory")
367 : #define rmb() asm volatile("lfence":::"memory")
368 : #define wmb() asm volatile("sfence" ::: "memory")
369 : #endif
370 :
371 : /**
372 : * read_barrier_depends - Flush all pending reads that subsequents reads
373 : * depend on.
374 : *
375 : * No data-dependent reads from memory-like regions are ever reordered
376 : * over this barrier. All reads preceding this primitive are guaranteed
377 : * to access memory (but not necessarily other CPUs' caches) before any
378 : * reads following this primitive that depend on the data return by
379 : * any of the preceding reads. This primitive is much lighter weight than
380 : * rmb() on most CPUs, and is never heavier weight than is
381 : * rmb().
382 : *
383 : * These ordering constraints are respected by both the local CPU
384 : * and the compiler.
385 : *
386 : * Ordering is not guaranteed by anything other than these primitives,
387 : * not even by data dependencies. See the documentation for
388 : * memory_barrier() for examples and URLs to more information.
389 : *
390 : * For example, the following code would force ordering (the initial
391 : * value of "a" is zero, "b" is one, and "p" is "&a"):
392 : *
393 : * <programlisting>
394 : * CPU 0 CPU 1
395 : *
396 : * b = 2;
397 : * memory_barrier();
398 : * p = &b; q = p;
399 : * read_barrier_depends();
400 : * d = *q;
401 : * </programlisting>
402 : *
403 : * because the read of "*q" depends on the read of "p" and these
404 : * two reads are separated by a read_barrier_depends(). However,
405 : * the following code, with the same initial values for "a" and "b":
406 : *
407 : * <programlisting>
408 : * CPU 0 CPU 1
409 : *
410 : * a = 2;
411 : * memory_barrier();
412 : * b = 3; y = b;
413 : * read_barrier_depends();
414 : * x = a;
415 : * </programlisting>
416 : *
417 : * does not enforce ordering, since there is no data dependency between
418 : * the read of "a" and the read of "b". Therefore, on some CPUs, such
419 : * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb()
420 : * in cases like this where there are no data dependencies.
421 : **/
422 :
423 : #define read_barrier_depends() do { } while (0)
424 :
425 : #ifdef CONFIG_SMP
426 : #define smp_mb() mb()
427 : #ifdef CONFIG_X86_PPRO_FENCE
428 : # define smp_rmb() rmb()
429 : #else
430 : # define smp_rmb() barrier()
431 : #endif
432 : #ifdef CONFIG_X86_OOSTORE
433 : # define smp_wmb() wmb()
434 : #else
435 : # define smp_wmb() barrier()
436 : #endif
437 : #define smp_read_barrier_depends() read_barrier_depends()
438 : #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
439 : #else
440 : #define smp_mb() barrier()
441 : #define smp_rmb() barrier()
442 : #define smp_wmb() barrier()
443 : #define smp_read_barrier_depends() do { } while (0)
444 : #define set_mb(var, value) do { var = value; barrier(); } while (0)
445 : #endif
446 :
447 : /*
448 : * Stop RDTSC speculation. This is needed when you need to use RDTSC
449 : * (or get_cycles or vread that possibly accesses the TSC) in a defined
450 : * code region.
451 : *
452 : * (Could use an alternative three way for this if there was one.)
453 : */
454 : static __always_inline void rdtsc_barrier(void)
455 : {
456 : alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
457 : alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
458 : }
459 1 :
460 : #endif /* _ASM_X86_SYSTEM_H */
|