1 /* $OpenBSD: kern_rwlock.c,v 1.13 2007/05/13 04:52:32 tedu Exp $ */
2
3 /*
4 * Copyright (c) 2002, 2003 Artur Grabowski <art@openbsd.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
17 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
18 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
19 * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/proc.h>
31 #include <sys/rwlock.h>
32 #include <sys/limits.h>
33
34 #include <machine/lock.h>
35
36 /* XXX - temporary measure until proc0 is properly aligned */
37 #define RW_PROC(p) (((long)p) & ~RWLOCK_MASK)
38
39 /*
40 * Magic wand for lock operations. Every operation checks if certain
41 * flags are set and if they aren't, it increments the lock with some
42 * value (that might need some computing in a few cases). If the operation
43 * fails, we need to set certain flags while waiting for the lock.
44 *
45 * RW_WRITE The lock must be completely empty. We increment it with
46 * RWLOCK_WRLOCK and the proc pointer of the holder.
47 * Sets RWLOCK_WAIT|RWLOCK_WRWANT while waiting.
48 * RW_READ RWLOCK_WRLOCK|RWLOCK_WRWANT may not be set. We increment
49 * with RWLOCK_READ_INCR. RWLOCK_WAIT while waiting.
50 */
51 static const struct rwlock_op {
52 unsigned long inc;
53 unsigned long check;
54 unsigned long wait_set;
55 long proc_mult;
56 int wait_prio;
57 } rw_ops[] = {
58 { /* RW_WRITE */
59 RWLOCK_WRLOCK,
60 ULONG_MAX,
61 RWLOCK_WAIT | RWLOCK_WRWANT,
62 1,
63 PLOCK - 4
64 },
65 { /* RW_READ */
66 RWLOCK_READ_INCR,
67 RWLOCK_WRLOCK,
68 RWLOCK_WAIT,
69 0,
70 PLOCK
71 },
72 { /* RW_DOWNGRADE */
73 RWLOCK_READ_INCR - RWLOCK_WRLOCK,
74 0,
75 0,
76 -1,
77 PLOCK
78 },
79 };
80
81 #ifndef __HAVE_MD_RWLOCK
82 /*
83 * Simple cases that should be in MD code and atomic.
84 */
85 void
86 rw_enter_read(struct rwlock *rwl)
87 {
88 unsigned long owner = rwl->rwl_owner;
89
90 if (__predict_false((owner & RWLOCK_WRLOCK) ||
91 rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR)))
92 rw_enter(rwl, RW_READ);
93 }
94
95 void
96 rw_enter_write(struct rwlock *rwl)
97 {
98 struct proc *p = curproc;
99
100 if (__predict_false(rw_cas(&rwl->rwl_owner, 0,
101 RW_PROC(p) | RWLOCK_WRLOCK)))
102 rw_enter(rwl, RW_WRITE);
103 }
104
105 void
106 rw_exit_read(struct rwlock *rwl)
107 {
108 unsigned long owner = rwl->rwl_owner;
109
110 if (__predict_false((owner & RWLOCK_WAIT) ||
111 rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR)))
112 rw_exit(rwl);
113 }
114
115 void
116 rw_exit_write(struct rwlock *rwl)
117 {
118 unsigned long owner = rwl->rwl_owner;
119
120 if (__predict_false((owner & RWLOCK_WAIT) ||
121 rw_cas(&rwl->rwl_owner, owner, 0)))
122 rw_exit(rwl);
123 }
124
125 #ifndef rw_cas
126 int
127 rw_cas(volatile unsigned long *p, unsigned long o, unsigned long n)
128 {
129 if (*p != o)
130 return (1);
131 *p = n;
132
133 return (0);
134 }
135 #endif
136
137 #endif
138
139 #ifdef DIAGNOSTIC
140 /*
141 * Put the diagnostic functions here to keep the main code free
142 * from ifdef clutter.
143 */
144 static void
145 rw_enter_diag(struct rwlock *rwl, int flags)
146 {
147 switch (flags & RW_OPMASK) {
148 case RW_WRITE:
149 case RW_READ:
150 if (RW_PROC(curproc) == RW_PROC(rwl->rwl_owner))
151 panic("rw_enter: %s locking against myself",
152 rwl->rwl_name);
153 break;
154 case RW_DOWNGRADE:
155 /*
156 * If we're downgrading, we must hold the write lock.
157 */
158 if ((rwl->rwl_owner & RWLOCK_WRLOCK) == 0)
159 panic("rw_enter: %s downgrade of non-write lock",
160 rwl->rwl_name);
161 if (RW_PROC(curproc) != RW_PROC(rwl->rwl_owner))
162 panic("rw_enter: %s downgrade, not holder",
163 rwl->rwl_name);
164 break;
165
166 default:
167 panic("rw_enter: unknown op 0x%x", flags);
168 }
169 }
170
171 #else
172 #define rw_enter_diag(r, f)
173 #endif
174
175 void
176 rw_init(struct rwlock *rwl, const char *name)
177 {
178 rwl->rwl_owner = 0;
179 rwl->rwl_name = name;
180 }
181
182 int
183 rw_enter(struct rwlock *rwl, int flags)
184 {
185 const struct rwlock_op *op;
186 struct sleep_state sls;
187 unsigned long inc, o;
188 int error;
189
190 op = &rw_ops[flags & RW_OPMASK];
191
192 inc = op->inc + RW_PROC(curproc) * op->proc_mult;
193 retry:
194 while (__predict_false(((o = rwl->rwl_owner) & op->check) != 0)) {
195 unsigned long set = o | op->wait_set;
196 int do_sleep;
197
198 rw_enter_diag(rwl, flags);
199
200 if (flags & RW_NOSLEEP)
201 return (EBUSY);
202
203 sleep_setup(&sls, rwl, op->wait_prio, rwl->rwl_name);
204 if (flags & RW_INTR)
205 sleep_setup_signal(&sls, op->wait_prio | PCATCH);
206
207 do_sleep = !rw_cas(&rwl->rwl_owner, o, set);
208
209 sleep_finish(&sls, do_sleep);
210 if ((flags & RW_INTR) &&
211 (error = sleep_finish_signal(&sls)) != 0)
212 return (error);
213 if (flags & RW_SLEEPFAIL)
214 return (EAGAIN);
215 }
216
217 if (__predict_false(rw_cas(&rwl->rwl_owner, o, o + inc)))
218 goto retry;
219
220 /*
221 * If old lock had RWLOCK_WAIT and RWLOCK_WRLOCK set, it means we
222 * downgraded a write lock and had possible read waiter, wake them
223 * to let them retry the lock.
224 */
225 if (__predict_false((o & (RWLOCK_WRLOCK|RWLOCK_WAIT)) ==
226 (RWLOCK_WRLOCK|RWLOCK_WAIT)))
227 wakeup(rwl);
228
229 return (0);
230 }
231
232 void
233 rw_exit(struct rwlock *rwl)
234 {
235 unsigned long owner = rwl->rwl_owner;
236 int wrlock = owner & RWLOCK_WRLOCK;
237 unsigned long set;
238
239 do {
240 owner = rwl->rwl_owner;
241 if (wrlock)
242 set = 0;
243 else
244 set = (owner - RWLOCK_READ_INCR) &
245 ~(RWLOCK_WAIT|RWLOCK_WRWANT);
246 } while (rw_cas(&rwl->rwl_owner, owner, set));
247
248 if (owner & RWLOCK_WAIT)
249 wakeup(rwl);
250 }