v2.4.9.9 -> v2.4.9.10
[opensuse:kernel.git] / include / linux / brlock.h
1 #ifndef __LINUX_BRLOCK_H
2 #define __LINUX_BRLOCK_H
3
4 /*
5  * 'Big Reader' read-write spinlocks.
6  *
7  * super-fast read/write locks, with write-side penalty. The point
8  * is to have a per-CPU read/write lock. Readers lock their CPU-local
9  * readlock, writers must lock all locks to get write access. These
10  * CPU-read-write locks are semantically identical to normal rwlocks.
11  * Memory usage is higher as well. (NR_CPUS*L1_CACHE_BYTES bytes)
12  *
13  * The most important feature is that these spinlocks do not cause
14  * cacheline ping-pong in the 'most readonly data' case.
15  *
16  * Copyright 2000, Ingo Molnar <mingo@redhat.com>
17  *
18  * Registry idea and naming [ crutial! :-) ] by:
19  *
20  *                 David S. Miller <davem@redhat.com>
21  *
22  * David has an implementation that doesnt use atomic operations in
23  * the read branch via memory ordering tricks - i guess we need to
24  * split this up into a per-arch thing? The atomicity issue is a
25  * secondary item in profiles, at least on x86 platforms.
26  *
27  * The atomic op version overhead is indeed a big deal on
28  * load-locked/store-conditional cpus (ALPHA/MIPS/PPC) and
29  * compare-and-swap cpus (Sparc64).  So we control which
30  * implementation to use with a __BRLOCK_USE_ATOMICS define. -DaveM
31  */
32
33 /* Register bigreader lock indices here. */
34 enum brlock_indices {
35         BR_GLOBALIRQ_LOCK,
36         BR_NETPROTO_LOCK,
37
38         __BR_END
39 };
40
41 #include <linux/config.h>
42
43 #ifdef CONFIG_SMP
44
45 #include <linux/cache.h>
46 #include <linux/spinlock.h>
47
48 #if defined(__i386__) || defined(__ia64__) || defined(__x86_64__)
49 #define __BRLOCK_USE_ATOMICS
50 #else
51 #undef __BRLOCK_USE_ATOMICS
52 #endif
53
54 #ifdef __BRLOCK_USE_ATOMICS
55 typedef rwlock_t        brlock_read_lock_t;
56 #else
57 typedef unsigned int    brlock_read_lock_t;
58 #endif
59
60 /*
61  * align last allocated index to the next cacheline:
62  */
63 #define __BR_IDX_MAX \
64         (((sizeof(brlock_read_lock_t)*__BR_END + SMP_CACHE_BYTES-1) & ~(SMP_CACHE_BYTES-1)) / sizeof(brlock_read_lock_t))
65
66 extern brlock_read_lock_t __brlock_array[NR_CPUS][__BR_IDX_MAX];
67
68 #ifndef __BRLOCK_USE_ATOMICS
69 struct br_wrlock {
70         spinlock_t lock;
71 } __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
72
73 extern struct br_wrlock __br_write_locks[__BR_IDX_MAX];
74 #endif
75
76 extern void __br_lock_usage_bug (void);
77
78 #ifdef __BRLOCK_USE_ATOMICS
79
80 static inline void br_read_lock (enum brlock_indices idx)
81 {
82         /*
83          * This causes a link-time bug message if an
84          * invalid index is used:
85          */
86         if (idx >= __BR_END)
87                 __br_lock_usage_bug();
88
89         read_lock(&__brlock_array[smp_processor_id()][idx]);
90 }
91
92 static inline void br_read_unlock (enum brlock_indices idx)
93 {
94         if (idx >= __BR_END)
95                 __br_lock_usage_bug();
96
97         read_unlock(&__brlock_array[smp_processor_id()][idx]);
98 }
99
100 #else /* ! __BRLOCK_USE_ATOMICS */
101 static inline void br_read_lock (enum brlock_indices idx)
102 {
103         unsigned int *ctr;
104         spinlock_t *lock;
105
106         /*
107          * This causes a link-time bug message if an
108          * invalid index is used:
109          */
110         if (idx >= __BR_END)
111                 __br_lock_usage_bug();
112
113         ctr = &__brlock_array[smp_processor_id()][idx];
114         lock = &__br_write_locks[idx].lock;
115 again:
116         (*ctr)++;
117         mb();
118         if (spin_is_locked(lock)) {
119                 (*ctr)--;
120                 wmb(); /*
121                         * The release of the ctr must become visible
122                         * to the other cpus eventually thus wmb(),
123                         * we don't care if spin_is_locked is reordered
124                         * before the releasing of the ctr.
125                         * However IMHO this wmb() is superflous even in theory.
126                         * It would not be superflous only if on the
127                         * other CPUs doing a ldl_l instead of an ldl
128                         * would make a difference and I don't think this is
129                         * the case.
130                         * I'd like to clarify this issue further
131                         * but for now this is a slow path so adding the
132                         * wmb() will keep us on the safe side.
133                         */
134                 while (spin_is_locked(lock))
135                         barrier();
136                 goto again;
137         }
138 }
139
140 static inline void br_read_unlock (enum brlock_indices idx)
141 {
142         unsigned int *ctr;
143
144         if (idx >= __BR_END)
145                 __br_lock_usage_bug();
146
147         ctr = &__brlock_array[smp_processor_id()][idx];
148
149         wmb();
150         (*ctr)--;
151 }
152 #endif /* __BRLOCK_USE_ATOMICS */
153
154 /* write path not inlined - it's rare and larger */
155
156 extern void FASTCALL(__br_write_lock (enum brlock_indices idx));
157 extern void FASTCALL(__br_write_unlock (enum brlock_indices idx));
158
159 static inline void br_write_lock (enum brlock_indices idx)
160 {
161         if (idx >= __BR_END)
162                 __br_lock_usage_bug();
163         __br_write_lock(idx);
164 }
165
166 static inline void br_write_unlock (enum brlock_indices idx)
167 {
168         if (idx >= __BR_END)
169                 __br_lock_usage_bug();
170         __br_write_unlock(idx);
171 }
172
173 #else
174 # define br_read_lock(idx)      ((void)(idx))
175 # define br_read_unlock(idx)    ((void)(idx))
176 # define br_write_lock(idx)     ((void)(idx))
177 # define br_write_unlock(idx)   ((void)(idx))
178 #endif
179
180 /*
181  * Now enumerate all of the possible sw/hw IRQ protected
182  * versions of the interfaces.
183  */
184 #define br_read_lock_irqsave(idx, flags) \
185         do { local_irq_save(flags); br_read_lock(idx); } while (0)
186
187 #define br_read_lock_irq(idx) \
188         do { local_irq_disable(); br_read_lock(idx); } while (0)
189
190 #define br_read_lock_bh(idx) \
191         do { local_bh_disable(); br_read_lock(idx); } while (0)
192
193 #define br_write_lock_irqsave(idx, flags) \
194         do { local_irq_save(flags); br_write_lock(idx); } while (0)
195
196 #define br_write_lock_irq(idx) \
197         do { local_irq_disable(); br_write_lock(idx); } while (0)
198
199 #define br_write_lock_bh(idx) \
200         do { local_bh_disable(); br_write_lock(idx); } while (0)
201
202 #define br_read_unlock_irqrestore(idx, flags) \
203         do { br_read_unlock(irx); local_irq_restore(flags); } while (0)
204
205 #define br_read_unlock_irq(idx) \
206         do { br_read_unlock(idx); local_irq_enable(); } while (0)
207
208 #define br_read_unlock_bh(idx) \
209         do { br_read_unlock(idx); local_bh_enable(); } while (0)
210
211 #define br_write_unlock_irqrestore(idx, flags) \
212         do { br_write_unlock(irx); local_irq_restore(flags); } while (0)
213
214 #define br_write_unlock_irq(idx) \
215         do { br_write_unlock(idx); local_irq_enable(); } while (0)
216
217 #define br_write_unlock_bh(idx) \
218         do { br_write_unlock(idx); local_bh_enable(); } while (0)
219
220 #endif /* __LINUX_BRLOCK_H */