Line data Source code
1 : #ifndef _ASM_X86_BITOPS_H
2 : #define _ASM_X86_BITOPS_H
3 :
4 : /*
5 : * Copyright 1992, Linus Torvalds.
6 : *
7 : * Note: inlines with more than a single statement should be marked
8 : * __always_inline to avoid problems with older gcc's inlining heuristics.
9 : */
10 :
11 : #ifndef _LINUX_BITOPS_H
12 : #error only <linux/bitops.h> can be included directly
13 : #endif
14 :
15 : #include <linux/compiler.h>
16 : #include <asm/alternative.h>
17 :
18 : /*
19 : * These have to be done with inline assembly: that way the bit-setting
20 : * is guaranteed to be atomic. All bit operations return 0 if the bit
21 : * was cleared before the operation and != 0 if it was not.
22 : *
23 : * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
24 : */
25 :
26 : #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
27 : /* Technically wrong, but this avoids compilation errors on some gcc
28 : versions. */
29 : #define BITOP_ADDR(x) "=m" (*(volatile long *) (x))
30 : #else
31 : #define BITOP_ADDR(x) "+m" (*(volatile long *) (x))
32 : #endif
33 :
34 : #define ADDR BITOP_ADDR(addr)
35 :
36 : /*
37 : * We do the locked ops that don't return the old value as
38 : * a mask operation on a byte.
39 : */
40 : #define IS_IMMEDIATE(nr) (__builtin_constant_p(nr))
41 : #define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((void *)(addr) + ((nr)>>3))
42 : #define CONST_MASK(nr) (1 << ((nr) & 7))
43 :
44 : /**
45 : * set_bit - Atomically set a bit in memory
46 : * @nr: the bit to set
47 : * @addr: the address to start counting from
48 : *
49 : * This function is atomic and may not be reordered. See __set_bit()
50 : * if you do not require the atomic guarantees.
51 : *
52 : * Note: there are no guarantees that this function will not be reordered
53 : * on non x86 architectures, so if you are writing portable code,
54 : * make sure not to rely on its reordering guarantees.
55 : *
56 : * Note that @nr may be almost arbitrarily large; this function is not
57 : * restricted to acting on a single-word quantity.
58 : */
59 : static __always_inline void
60 : set_bit(unsigned int nr, volatile unsigned long *addr)
61 : {
62 : if (IS_IMMEDIATE(nr)) {
63 : asm volatile(LOCK_PREFIX "orb %1,%0"
64 : : CONST_MASK_ADDR(nr, addr)
65 : : "iq" ((u8)CONST_MASK(nr))
66 : : "memory");
67 : } else {
68 : asm volatile(LOCK_PREFIX "bts %1,%0"
69 : : BITOP_ADDR(addr) : "Ir" (nr) : "memory");
70 : }
71 : }
72 :
73 : /**
74 : * __set_bit - Set a bit in memory
75 : * @nr: the bit to set
76 : * @addr: the address to start counting from
77 : *
78 : * Unlike set_bit(), this function is non-atomic and may be reordered.
79 : * If it's called on the same region of memory simultaneously, the effect
80 : * may be that only one operation succeeds.
81 : */
82 : static inline void __set_bit(int nr, volatile unsigned long *addr)
83 : {
84 : asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
85 : }
86 :
87 : /**
88 : * clear_bit - Clears a bit in memory
89 : * @nr: Bit to clear
90 : * @addr: Address to start counting from
91 : *
92 : * clear_bit() is atomic and may not be reordered. However, it does
93 : * not contain a memory barrier, so if it is used for locking purposes,
94 : * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
95 : * in order to ensure changes are visible on other processors.
96 : */
97 : static __always_inline void
98 : clear_bit(int nr, volatile unsigned long *addr)
99 : {
100 : if (IS_IMMEDIATE(nr)) {
101 : asm volatile(LOCK_PREFIX "andb %1,%0"
102 : : CONST_MASK_ADDR(nr, addr)
103 : : "iq" ((u8)~CONST_MASK(nr)));
104 : } else {
105 : asm volatile(LOCK_PREFIX "btr %1,%0"
106 : : BITOP_ADDR(addr)
107 : : "Ir" (nr));
108 : }
109 : }
110 :
111 : /*
112 : * clear_bit_unlock - Clears a bit in memory
113 : * @nr: Bit to clear
114 : * @addr: Address to start counting from
115 : *
116 : * clear_bit() is atomic and implies release semantics before the memory
117 : * operation. It can be used for an unlock.
118 : */
119 : static inline void clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
120 : {
121 : barrier();
122 : clear_bit(nr, addr);
123 : }
124 :
125 : static inline void __clear_bit(int nr, volatile unsigned long *addr)
126 : {
127 : asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
128 : }
129 :
130 : /*
131 : * __clear_bit_unlock - Clears a bit in memory
132 : * @nr: Bit to clear
133 : * @addr: Address to start counting from
134 : *
135 : * __clear_bit() is non-atomic and implies release semantics before the memory
136 : * operation. It can be used for an unlock if no other CPUs can concurrently
137 : * modify other bits in the word.
138 : *
139 : * No memory barrier is required here, because x86 cannot reorder stores past
140 : * older loads. Same principle as spin_unlock.
141 : */
142 : static inline void __clear_bit_unlock(unsigned nr, volatile unsigned long *addr)
143 : {
144 : barrier();
145 : __clear_bit(nr, addr);
146 : }
147 :
148 : #define smp_mb__before_clear_bit() barrier()
149 : #define smp_mb__after_clear_bit() barrier()
150 :
151 : /**
152 : * __change_bit - Toggle a bit in memory
153 : * @nr: the bit to change
154 : * @addr: the address to start counting from
155 : *
156 : * Unlike change_bit(), this function is non-atomic and may be reordered.
157 : * If it's called on the same region of memory simultaneously, the effect
158 : * may be that only one operation succeeds.
159 : */
160 : static inline void __change_bit(int nr, volatile unsigned long *addr)
161 : {
162 : asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
163 : }
164 :
165 : /**
166 : * change_bit - Toggle a bit in memory
167 : * @nr: Bit to change
168 : * @addr: Address to start counting from
169 : *
170 : * change_bit() is atomic and may not be reordered.
171 : * Note that @nr may be almost arbitrarily large; this function is not
172 : * restricted to acting on a single-word quantity.
173 : */
174 : static inline void change_bit(int nr, volatile unsigned long *addr)
175 : {
176 : if (IS_IMMEDIATE(nr)) {
177 : asm volatile(LOCK_PREFIX "xorb %1,%0"
178 : : CONST_MASK_ADDR(nr, addr)
179 : : "iq" ((u8)CONST_MASK(nr)));
180 : } else {
181 : asm volatile(LOCK_PREFIX "btc %1,%0"
182 : : BITOP_ADDR(addr)
183 : : "Ir" (nr));
184 : }
185 : }
186 :
187 : /**
188 : * test_and_set_bit - Set a bit and return its old value
189 : * @nr: Bit to set
190 : * @addr: Address to count from
191 : *
192 : * This operation is atomic and cannot be reordered.
193 : * It also implies a memory barrier.
194 : */
195 : static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
196 : {
197 : int oldbit;
198 :
199 : asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
200 : "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
201 :
202 : return oldbit;
203 : }
204 :
205 : /**
206 : * test_and_set_bit_lock - Set a bit and return its old value for lock
207 : * @nr: Bit to set
208 : * @addr: Address to count from
209 : *
210 : * This is the same as test_and_set_bit on x86.
211 : */
212 : static __always_inline int
213 : test_and_set_bit_lock(int nr, volatile unsigned long *addr)
214 : {
215 : return test_and_set_bit(nr, addr);
216 : }
217 :
218 : /**
219 : * __test_and_set_bit - Set a bit and return its old value
220 : * @nr: Bit to set
221 : * @addr: Address to count from
222 : *
223 : * This operation is non-atomic and can be reordered.
224 : * If two examples of this operation race, one can appear to succeed
225 : * but actually fail. You must protect multiple accesses with a lock.
226 : */
227 : static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
228 : {
229 : int oldbit;
230 :
231 : asm("bts %2,%1\n\t"
232 : "sbb %0,%0"
233 : : "=r" (oldbit), ADDR
234 : : "Ir" (nr));
235 : return oldbit;
236 : }
237 :
238 : /**
239 : * test_and_clear_bit - Clear a bit and return its old value
240 : * @nr: Bit to clear
241 : * @addr: Address to count from
242 : *
243 : * This operation is atomic and cannot be reordered.
244 : * It also implies a memory barrier.
245 : */
246 : static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
247 : {
248 : int oldbit;
249 :
250 : asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
251 : "sbb %0,%0"
252 : : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
253 :
254 : return oldbit;
255 : }
256 :
257 : /**
258 : * __test_and_clear_bit - Clear a bit and return its old value
259 : * @nr: Bit to clear
260 : * @addr: Address to count from
261 : *
262 : * This operation is non-atomic and can be reordered.
263 : * If two examples of this operation race, one can appear to succeed
264 : * but actually fail. You must protect multiple accesses with a lock.
265 : */
266 : static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
267 : {
268 : int oldbit;
269 :
270 : asm volatile("btr %2,%1\n\t"
271 : "sbb %0,%0"
272 : : "=r" (oldbit), ADDR
273 : : "Ir" (nr));
274 : return oldbit;
275 : }
276 :
277 : /* WARNING: non atomic and it can be reordered! */
278 : static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
279 : {
280 : int oldbit;
281 :
282 : asm volatile("btc %2,%1\n\t"
283 : "sbb %0,%0"
284 : : "=r" (oldbit), ADDR
285 : : "Ir" (nr) : "memory");
286 :
287 : return oldbit;
288 : }
289 :
290 : /**
291 : * test_and_change_bit - Change a bit and return its old value
292 : * @nr: Bit to change
293 : * @addr: Address to count from
294 : *
295 : * This operation is atomic and cannot be reordered.
296 : * It also implies a memory barrier.
297 : */
298 : static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
299 : {
300 : int oldbit;
301 :
302 : asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
303 : "sbb %0,%0"
304 : : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
305 :
306 : return oldbit;
307 : }
308 :
309 : static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
310 : {
311 0 : return ((1UL << (nr % BITS_PER_LONG)) &
312 : (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
313 : }
314 :
315 : static inline int variable_test_bit(int nr, volatile const unsigned long *addr)
316 : {
317 : int oldbit;
318 :
319 : asm volatile("bt %2,%1\n\t"
320 : "sbb %0,%0"
321 : : "=r" (oldbit)
322 : : "m" (*(unsigned long *)addr), "Ir" (nr));
323 :
324 : return oldbit;
325 : }
326 :
327 : #if 0 /* Fool kernel-doc since it doesn't do macros yet */
328 : /**
329 : * test_bit - Determine whether a bit is set
330 : * @nr: bit number to test
331 : * @addr: Address to start counting from
332 : */
333 : static int test_bit(int nr, const volatile unsigned long *addr);
334 : #endif
335 :
336 : #define test_bit(nr, addr) \
337 : (__builtin_constant_p((nr)) \
338 : ? constant_test_bit((nr), (addr)) \
339 : : variable_test_bit((nr), (addr)))
340 :
341 : /**
342 : * __ffs - find first set bit in word
343 : * @word: The word to search
344 : *
345 : * Undefined if no bit exists, so code should check against 0 first.
346 : */
347 : static inline unsigned long __ffs(unsigned long word)
348 : {
349 : asm("bsf %1,%0"
350 : : "=r" (word)
351 : : "rm" (word));
352 : return word;
353 : }
354 :
355 : /**
356 : * ffz - find first zero bit in word
357 : * @word: The word to search
358 : *
359 : * Undefined if no zero exists, so code should check against ~0UL first.
360 : */
361 : static inline unsigned long ffz(unsigned long word)
362 : {
363 : asm("bsf %1,%0"
364 : : "=r" (word)
365 : : "r" (~word));
366 : return word;
367 : }
368 :
369 : /*
370 : * __fls: find last set bit in word
371 : * @word: The word to search
372 : *
373 : * Undefined if no set bit exists, so code should check against 0 first.
374 : */
375 : static inline unsigned long __fls(unsigned long word)
376 : {
377 : asm("bsr %1,%0"
378 : : "=r" (word)
379 : : "rm" (word));
380 : return word;
381 : }
382 :
383 : #ifdef __KERNEL__
384 : /**
385 : * ffs - find first set bit in word
386 : * @x: the word to search
387 : *
388 : * This is defined the same way as the libc and compiler builtin ffs
389 : * routines, therefore differs in spirit from the other bitops.
390 : *
391 : * ffs(value) returns 0 if value is 0 or the position of the first
392 : * set bit if value is nonzero. The first (least significant) bit
393 : * is at position 1.
394 : */
395 : static inline int ffs(int x)
396 : {
397 : int r;
398 : #ifdef CONFIG_X86_CMOV
399 : asm("bsfl %1,%0\n\t"
400 : "cmovzl %2,%0"
401 : : "=r" (r) : "rm" (x), "r" (-1));
402 : #else
403 : asm("bsfl %1,%0\n\t"
404 : "jnz 1f\n\t"
405 : "movl $-1,%0\n"
406 : "1:" : "=r" (r) : "rm" (x));
407 : #endif
408 : return r + 1;
409 : }
410 :
411 : /**
412 : * fls - find last set bit in word
413 : * @x: the word to search
414 : *
415 : * This is defined in a similar way as the libc and compiler builtin
416 : * ffs, but returns the position of the most significant set bit.
417 : *
418 : * fls(value) returns 0 if value is 0 or the position of the last
419 : * set bit if value is nonzero. The last (most significant) bit is
420 : * at position 32.
421 : */
422 : static inline int fls(int x)
423 : {
424 : int r;
425 : #ifdef CONFIG_X86_CMOV
426 : asm("bsrl %1,%0\n\t"
427 : "cmovzl %2,%0"
428 : : "=&r" (r) : "rm" (x), "rm" (-1));
429 : #else
430 : asm("bsrl %1,%0\n\t"
431 : "jnz 1f\n\t"
432 : "movl $-1,%0\n"
433 : "1:" : "=r" (r) : "rm" (x));
434 : #endif
435 : return r + 1;
436 : }
437 : #endif /* __KERNEL__ */
438 :
439 : #undef ADDR
440 :
441 : #ifdef __KERNEL__
442 :
443 : #include <asm-generic/bitops/sched.h>
444 :
445 : #define ARCH_HAS_FAST_MULTIPLIER 1
446 :
447 : #include <asm-generic/bitops/hweight.h>
448 :
449 : #endif /* __KERNEL__ */
450 :
451 : #include <asm-generic/bitops/fls64.h>
452 :
453 : #ifdef __KERNEL__
454 :
455 : #include <asm-generic/bitops/ext2-non-atomic.h>
456 :
457 : #define ext2_set_bit_atomic(lock, nr, addr) \
458 : test_and_set_bit((nr), (unsigned long *)(addr))
459 : #define ext2_clear_bit_atomic(lock, nr, addr) \
460 : test_and_clear_bit((nr), (unsigned long *)(addr))
461 :
462 : #include <asm-generic/bitops/minix.h>
463 :
464 : #endif /* __KERNEL__ */
465 : #endif /* _ASM_X86_BITOPS_H */
|