1 /* $OpenBSD: uvm_vnode.c,v 1.49 2007/06/05 00:38:24 deraadt Exp $ */
2 /* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */
3
4 /*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * Copyright (c) 1991, 1993
7 * The Regents of the University of California.
8 * Copyright (c) 1990 University of Utah.
9 *
10 * All rights reserved.
11 *
12 * This code is derived from software contributed to Berkeley by
13 * the Systems Programming Group of the University of Utah Computer
14 * Science Department.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. All advertising materials mentioning features or use of this software
25 * must display the following acknowledgement:
26 * This product includes software developed by Charles D. Cranor,
27 * Washington University, the University of California, Berkeley and
28 * its contributors.
29 * 4. Neither the name of the University nor the names of its contributors
30 * may be used to endorse or promote products derived from this software
31 * without specific prior written permission.
32 *
33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * SUCH DAMAGE.
44 *
45 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94
46 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp
47 */
48
49 /*
50 * uvm_vnode.c: the vnode pager.
51 */
52
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/proc.h>
56 #include <sys/malloc.h>
57 #include <sys/vnode.h>
58 #include <sys/disklabel.h>
59 #include <sys/ioctl.h>
60 #include <sys/fcntl.h>
61 #include <sys/conf.h>
62 #include <sys/rwlock.h>
63
64 #include <miscfs/specfs/specdev.h>
65
66 #include <uvm/uvm.h>
67 #include <uvm/uvm_vnode.h>
68
69 /*
70 * private global data structure
71 *
72 * we keep a list of writeable active vnode-backed VM objects for sync op.
73 * we keep a simpleq of vnodes that are currently being sync'd.
74 */
75
76 LIST_HEAD(uvn_list_struct, uvm_vnode);
77 struct uvn_list_struct uvn_wlist; /* writeable uvns */
78
79 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode);
80 struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */
81 struct rwlock uvn_sync_lock; /* locks sync operation */
82
83 /*
84 * functions
85 */
86
87 void uvn_cluster(struct uvm_object *, voff_t,
88 voff_t *, voff_t *);
89 void uvn_detach(struct uvm_object *);
90 boolean_t uvn_flush(struct uvm_object *, voff_t,
91 voff_t, int);
92 int uvn_get(struct uvm_object *, voff_t,
93 vm_page_t *, int *, int,
94 vm_prot_t, int, int);
95 void uvn_init(void);
96 int uvn_io(struct uvm_vnode *, vm_page_t *,
97 int, int, int);
98 int uvn_put(struct uvm_object *, vm_page_t *,
99 int, boolean_t);
100 void uvn_reference(struct uvm_object *);
101 boolean_t uvn_releasepg(struct vm_page *,
102 struct vm_page **);
103
104 /*
105 * master pager structure
106 */
107
108 struct uvm_pagerops uvm_vnodeops = {
109 uvn_init,
110 uvn_reference,
111 uvn_detach,
112 NULL, /* no specialized fault routine required */
113 uvn_flush,
114 uvn_get,
115 uvn_put,
116 uvn_cluster,
117 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */
118 uvn_releasepg,
119 };
120
121 /*
122 * the ops!
123 */
124
125 /*
126 * uvn_init
127 *
128 * init pager private data structures.
129 */
130
131 void
132 uvn_init(void)
133 {
134
135 LIST_INIT(&uvn_wlist);
136 /* note: uvn_sync_q init'd in uvm_vnp_sync() */
137 rw_init(&uvn_sync_lock, "uvnsync");
138 }
139
140 /*
141 * uvn_attach
142 *
143 * attach a vnode structure to a VM object. if the vnode is already
144 * attached, then just bump the reference count by one and return the
145 * VM object. if not already attached, attach and return the new VM obj.
146 * the "accessprot" tells the max access the attaching thread wants to
147 * our pages.
148 *
149 * => caller must _not_ already be holding the lock on the uvm_object.
150 * => in fact, nothing should be locked so that we can sleep here.
151 * => note that uvm_object is first thing in vnode structure, so their
152 * pointers are equiv.
153 */
154
155 struct uvm_object *
156 uvn_attach(arg, accessprot)
157 void *arg;
158 vm_prot_t accessprot;
159 {
160 struct vnode *vp = arg;
161 struct uvm_vnode *uvn = &vp->v_uvm;
162 struct vattr vattr;
163 int oldflags, result;
164 struct partinfo pi;
165 u_quad_t used_vnode_size;
166 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist);
167
168 UVMHIST_LOG(maphist, "(vn=%p)", arg,0,0,0);
169
170 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */
171
172 /*
173 * first get a lock on the uvn.
174 */
175 simple_lock(&uvn->u_obj.vmobjlock);
176 while (uvn->u_flags & UVM_VNODE_BLOCKED) {
177 printf("uvn_attach: blocked at %p flags 0x%x\n",
178 uvn, uvn->u_flags);
179 uvn->u_flags |= UVM_VNODE_WANTED;
180 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0);
181 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE,
182 "uvn_attach", 0);
183 simple_lock(&uvn->u_obj.vmobjlock);
184 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0);
185 }
186
187 /*
188 * if we're mapping a BLK device, make sure it is a disk.
189 */
190 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) {
191 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */
192 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0);
193 return(NULL);
194 }
195
196 /*
197 * now we have lock and uvn must not be in a blocked state.
198 * first check to see if it is already active, in which case
199 * we can bump the reference count, check to see if we need to
200 * add it to the writeable list, and then return.
201 */
202 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */
203
204 /* regain VREF if we were persisting */
205 if (uvn->u_obj.uo_refs == 0) {
206 VREF(vp);
207 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)",
208 0,0,0,0);
209 }
210 uvn->u_obj.uo_refs++; /* bump uvn ref! */
211
212 /* check for new writeable uvn */
213 if ((accessprot & VM_PROT_WRITE) != 0 &&
214 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) {
215 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist);
216 /* we are now on wlist! */
217 uvn->u_flags |= UVM_VNODE_WRITEABLE;
218 }
219
220 /* unlock and return */
221 simple_unlock(&uvn->u_obj.vmobjlock);
222 UVMHIST_LOG(maphist,"<- done, refcnt=%ld", uvn->u_obj.uo_refs,
223 0, 0, 0);
224 return (&uvn->u_obj);
225 }
226
227 /*
228 * need to call VOP_GETATTR() to get the attributes, but that could
229 * block (due to I/O), so we want to unlock the object before calling.
230 * however, we want to keep anyone else from playing with the object
231 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which
232 * prevents anyone from attaching to the vnode until we are done with
233 * it.
234 */
235 uvn->u_flags = UVM_VNODE_ALOCK;
236 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */
237 /* XXX: curproc? */
238
239 if (vp->v_type == VBLK) {
240 /*
241 * We could implement this as a specfs getattr call, but:
242 *
243 * (1) VOP_GETATTR() would get the file system
244 * vnode operation, not the specfs operation.
245 *
246 * (2) All we want is the size, anyhow.
247 */
248 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev,
249 DIOCGPART, (caddr_t)&pi, FREAD, curproc);
250 if (result == 0) {
251 /* XXX should remember blocksize */
252 used_vnode_size = (u_quad_t)pi.disklab->d_secsize *
253 (u_quad_t)DL_GETPSIZE(pi.part);
254 }
255 } else {
256 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc);
257 if (result == 0)
258 used_vnode_size = vattr.va_size;
259 }
260
261 /* relock object */
262 simple_lock(&uvn->u_obj.vmobjlock);
263
264 if (result != 0) {
265 if (uvn->u_flags & UVM_VNODE_WANTED)
266 wakeup(uvn);
267 uvn->u_flags = 0;
268 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */
269 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0);
270 return(NULL);
271 }
272
273 /*
274 * make sure that the newsize fits within a vaddr_t
275 * XXX: need to revise addressing data types
276 */
277 #ifdef DEBUG
278 if (vp->v_type == VBLK)
279 printf("used_vnode_size = %llu\n", (long long)used_vnode_size);
280 #endif
281
282 /*
283 * now set up the uvn.
284 */
285 uvn->u_obj.pgops = &uvm_vnodeops;
286 TAILQ_INIT(&uvn->u_obj.memq);
287 uvn->u_obj.uo_npages = 0;
288 uvn->u_obj.uo_refs = 1; /* just us... */
289 oldflags = uvn->u_flags;
290 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST;
291 uvn->u_nio = 0;
292 uvn->u_size = used_vnode_size;
293
294 /* if write access, we need to add it to the wlist */
295 if (accessprot & VM_PROT_WRITE) {
296 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist);
297 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */
298 }
299
300 /*
301 * add a reference to the vnode. this reference will stay as long
302 * as there is a valid mapping of the vnode. dropped when the
303 * reference count goes to zero [and we either free or persist].
304 */
305 VREF(vp);
306 simple_unlock(&uvn->u_obj.vmobjlock);
307 if (oldflags & UVM_VNODE_WANTED)
308 wakeup(uvn);
309
310 UVMHIST_LOG(maphist,"<- done/VREF, ret %p", &uvn->u_obj,0,0,0);
311 return(&uvn->u_obj);
312 }
313
314
315 /*
316 * uvn_reference
317 *
318 * duplicate a reference to a VM object. Note that the reference
319 * count must already be at least one (the passed in reference) so
320 * there is no chance of the uvn being killed or locked out here.
321 *
322 * => caller must call with object unlocked.
323 * => caller must be using the same accessprot as was used at attach time
324 */
325
326
327 void
328 uvn_reference(uobj)
329 struct uvm_object *uobj;
330 {
331 #ifdef DEBUG
332 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
333 #endif
334 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist);
335
336 simple_lock(&uobj->vmobjlock);
337 #ifdef DEBUG
338 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) {
339 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags,
340 uobj->uo_refs);
341 panic("uvn_reference: invalid state");
342 }
343 #endif
344 uobj->uo_refs++;
345 UVMHIST_LOG(maphist, "<- done (uobj=%p, ref = %ld)",
346 uobj, uobj->uo_refs,0,0);
347 simple_unlock(&uobj->vmobjlock);
348 }
349
350 /*
351 * uvn_detach
352 *
353 * remove a reference to a VM object.
354 *
355 * => caller must call with object unlocked and map locked.
356 * => this starts the detach process, but doesn't have to finish it
357 * (async i/o could still be pending).
358 */
359 void
360 uvn_detach(uobj)
361 struct uvm_object *uobj;
362 {
363 struct uvm_vnode *uvn;
364 struct vnode *vp;
365 int oldflags;
366 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist);
367
368 simple_lock(&uobj->vmobjlock);
369
370 UVMHIST_LOG(maphist," (uobj=%p) ref=%ld", uobj,uobj->uo_refs,0,0);
371 uobj->uo_refs--; /* drop ref! */
372 if (uobj->uo_refs) { /* still more refs */
373 simple_unlock(&uobj->vmobjlock);
374 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
375 return;
376 }
377
378 /*
379 * get other pointers ...
380 */
381
382 uvn = (struct uvm_vnode *) uobj;
383 vp = (struct vnode *) uobj;
384
385 /*
386 * clear VTEXT flag now that there are no mappings left (VTEXT is used
387 * to keep an active text file from being overwritten).
388 */
389 vp->v_flag &= ~VTEXT;
390
391 /*
392 * we just dropped the last reference to the uvn. see if we can
393 * let it "stick around".
394 */
395
396 if (uvn->u_flags & UVM_VNODE_CANPERSIST) {
397 /* won't block */
398 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES);
399 simple_unlock(&uobj->vmobjlock);
400 vrele(vp); /* drop vnode reference */
401 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0);
402 return;
403 }
404
405 /*
406 * its a goner!
407 */
408
409 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0);
410
411 uvn->u_flags |= UVM_VNODE_DYING;
412
413 /*
414 * even though we may unlock in flush, no one can gain a reference
415 * to us until we clear the "dying" flag [because it blocks
416 * attaches]. we will not do that until after we've disposed of all
417 * the pages with uvn_flush(). note that before the flush the only
418 * pages that could be marked PG_BUSY are ones that are in async
419 * pageout by the daemon. (there can't be any pending "get"'s
420 * because there are no references to the object).
421 */
422
423 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES);
424
425 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0);
426
427 /*
428 * given the structure of this pager, the above flush request will
429 * create the following state: all the pages that were in the object
430 * have either been free'd or they are marked PG_BUSY|PG_RELEASED.
431 * the PG_BUSY bit was set either by us or the daemon for async I/O.
432 * in either case, if we have pages left we can't kill the object
433 * yet because i/o is pending. in this case we set the "relkill"
434 * flag which will cause pgo_releasepg to kill the object once all
435 * the I/O's are done [pgo_releasepg will be called from the aiodone
436 * routine or from the page daemon].
437 */
438
439 if (uobj->uo_npages) { /* I/O pending. iodone will free */
440 #ifdef DEBUG
441 /*
442 * XXXCDC: very unlikely to happen until we have async i/o
443 * so print a little info message in case it does.
444 */
445 printf("uvn_detach: vn %p has pages left after flush - "
446 "relkill mode\n", uobj);
447 #endif
448 uvn->u_flags |= UVM_VNODE_RELKILL;
449 simple_unlock(&uobj->vmobjlock);
450 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0,
451 0, 0);
452 return;
453 }
454
455 /*
456 * kill object now. note that we can't be on the sync q because
457 * all references are gone.
458 */
459 if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
460 LIST_REMOVE(uvn, u_wlist);
461 }
462 #ifdef DIAGNOSTIC
463 if (!TAILQ_EMPTY(&uobj->memq))
464 panic("uvn_deref: vnode VM object still has pages afer "
465 "syncio/free flush");
466 #endif
467 oldflags = uvn->u_flags;
468 uvn->u_flags = 0;
469 simple_unlock(&uobj->vmobjlock);
470
471 /* wake up any sleepers */
472 if (oldflags & UVM_VNODE_WANTED)
473 wakeup(uvn);
474
475 /*
476 * drop our reference to the vnode.
477 */
478 vrele(vp);
479 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0);
480
481 return;
482 }
483
484 /*
485 * uvm_vnp_terminate: external hook to clear out a vnode's VM
486 *
487 * called in two cases:
488 * [1] when a persisting vnode vm object (i.e. one with a zero reference
489 * count) needs to be freed so that a vnode can be reused. this
490 * happens under "getnewvnode" in vfs_subr.c. if the vnode from
491 * the free list is still attached (i.e. not VBAD) then vgone is
492 * called. as part of the vgone trace this should get called to
493 * free the vm object. this is the common case.
494 * [2] when a filesystem is being unmounted by force (MNT_FORCE,
495 * "umount -f") the vgone() function is called on active vnodes
496 * on the mounted file systems to kill their data (the vnodes become
497 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a
498 * call here (even if the uvn is still in use -- i.e. has a non-zero
499 * reference count). this case happens at "umount -f" and during a
500 * "reboot/halt" operation.
501 *
502 * => the caller must XLOCK and VOP_LOCK the vnode before calling us
503 * [protects us from getting a vnode that is already in the DYING
504 * state...]
505 * => unlike uvn_detach, this function must not return until all the
506 * uvn's pages are disposed of.
507 * => in case [2] the uvn is still alive after this call, but all I/O
508 * ops will fail (due to the backing vnode now being "dead"). this
509 * will prob. kill any process using the uvn due to pgo_get failing.
510 */
511
512 void
513 uvm_vnp_terminate(vp)
514 struct vnode *vp;
515 {
516 struct uvm_vnode *uvn = &vp->v_uvm;
517 int oldflags;
518 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist);
519
520 /*
521 * lock object and check if it is valid
522 */
523 simple_lock(&uvn->u_obj.vmobjlock);
524 UVMHIST_LOG(maphist, " vp=%p, ref=%ld, flag=0x%lx", vp,
525 uvn->u_obj.uo_refs, uvn->u_flags, 0);
526 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) {
527 simple_unlock(&uvn->u_obj.vmobjlock);
528 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0);
529 return;
530 }
531
532 /*
533 * must be a valid uvn that is not already dying (because XLOCK
534 * protects us from that). the uvn can't in the ALOCK state
535 * because it is valid, and uvn's that are in the ALOCK state haven't
536 * been marked valid yet.
537 */
538
539 #ifdef DEBUG
540 /*
541 * debug check: are we yanking the vnode out from under our uvn?
542 */
543 if (uvn->u_obj.uo_refs) {
544 printf("uvm_vnp_terminate(%p): terminating active vnode "
545 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs);
546 }
547 #endif
548
549 /*
550 * it is possible that the uvn was detached and is in the relkill
551 * state [i.e. waiting for async i/o to finish so that releasepg can
552 * kill object]. we take over the vnode now and cancel the relkill.
553 * we want to know when the i/o is done so we can recycle right
554 * away. note that a uvn can only be in the RELKILL state if it
555 * has a zero reference count.
556 */
557
558 if (uvn->u_flags & UVM_VNODE_RELKILL)
559 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */
560
561 /*
562 * block the uvn by setting the dying flag, and then flush the
563 * pages. (note that flush may unlock object while doing I/O, but
564 * it will re-lock it before it returns control here).
565 *
566 * also, note that we tell I/O that we are already VOP_LOCK'd so
567 * that uvn_io doesn't attempt to VOP_LOCK again.
568 *
569 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated
570 * due to a forceful unmount might not be a good idea. maybe we
571 * need a way to pass in this info to uvn_flush through a
572 * pager-defined PGO_ constant [currently there are none].
573 */
574 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED;
575
576 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES);
577
578 /*
579 * as we just did a flush we expect all the pages to be gone or in
580 * the process of going. sleep to wait for the rest to go [via iosync].
581 */
582
583 while (uvn->u_obj.uo_npages) {
584 #ifdef DEBUG
585 struct vm_page *pp;
586 TAILQ_FOREACH(pp, &uvn->u_obj.memq, listq) {
587 if ((pp->pg_flags & PG_BUSY) == 0)
588 panic("uvm_vnp_terminate: detected unbusy pg");
589 }
590 if (uvn->u_nio == 0)
591 panic("uvm_vnp_terminate: no I/O to wait for?");
592 printf("uvm_vnp_terminate: waiting for I/O to fin.\n");
593 /*
594 * XXXCDC: this is unlikely to happen without async i/o so we
595 * put a printf in just to keep an eye on it.
596 */
597 #endif
598 uvn->u_flags |= UVM_VNODE_IOSYNC;
599 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE,
600 "uvn_term",0);
601 simple_lock(&uvn->u_obj.vmobjlock);
602 }
603
604 /*
605 * done. now we free the uvn if its reference count is zero
606 * (true if we are zapping a persisting uvn). however, if we are
607 * terminating a uvn with active mappings we let it live ... future
608 * calls down to the vnode layer will fail.
609 */
610
611 oldflags = uvn->u_flags;
612 if (uvn->u_obj.uo_refs) {
613
614 /*
615 * uvn must live on it is dead-vnode state until all references
616 * are gone. restore flags. clear CANPERSIST state.
617 */
618
619 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED|
620 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST);
621
622 } else {
623
624 /*
625 * free the uvn now. note that the VREF reference is already
626 * gone [it is dropped when we enter the persist state].
627 */
628 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED)
629 panic("uvm_vnp_terminate: io sync wanted bit set");
630
631 if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
632 LIST_REMOVE(uvn, u_wlist);
633 }
634 uvn->u_flags = 0; /* uvn is history, clear all bits */
635 }
636
637 if (oldflags & UVM_VNODE_WANTED)
638 wakeup(uvn); /* object lock still held */
639
640 simple_unlock(&uvn->u_obj.vmobjlock);
641 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0);
642
643 }
644
645 /*
646 * uvn_releasepg: handled a released page in a uvn
647 *
648 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
649 * to dispose of.
650 * => caller must handled PG_WANTED case
651 * => called with page's object locked, pageq's unlocked
652 * => returns TRUE if page's object is still alive, FALSE if we
653 * killed the page's object. if we return TRUE, then we
654 * return with the object locked.
655 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return
656 * with the page queues locked [for pagedaemon]
657 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
658 * => we kill the uvn if it is not referenced and we are suppose to
659 * kill it ("relkill").
660 */
661
662 boolean_t
663 uvn_releasepg(pg, nextpgp)
664 struct vm_page *pg;
665 struct vm_page **nextpgp; /* OUT */
666 {
667 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject;
668 #ifdef DIAGNOSTIC
669 if ((pg->pg_flags & PG_RELEASED) == 0)
670 panic("uvn_releasepg: page not released!");
671 #endif
672
673 /*
674 * dispose of the page [caller handles PG_WANTED]
675 */
676 pmap_page_protect(pg, VM_PROT_NONE);
677 uvm_lock_pageq();
678 if (nextpgp)
679 *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
680 uvm_pagefree(pg);
681 if (!nextpgp)
682 uvm_unlock_pageq();
683
684 /*
685 * now see if we need to kill the object
686 */
687 if (uvn->u_flags & UVM_VNODE_RELKILL) {
688 if (uvn->u_obj.uo_refs)
689 panic("uvn_releasepg: kill flag set on referenced "
690 "object!");
691 if (uvn->u_obj.uo_npages == 0) {
692 if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
693 LIST_REMOVE(uvn, u_wlist);
694 }
695 #ifdef DIAGNOSTIC
696 if (!TAILQ_EMPTY(&uvn->u_obj.memq))
697 panic("uvn_releasepg: pages in object with npages == 0");
698 #endif
699 if (uvn->u_flags & UVM_VNODE_WANTED)
700 /* still holding object lock */
701 wakeup(uvn);
702
703 uvn->u_flags = 0; /* DEAD! */
704 simple_unlock(&uvn->u_obj.vmobjlock);
705 return (FALSE);
706 }
707 }
708 return (TRUE);
709 }
710
711 /*
712 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go
713 * through the buffer cache and allow I/O in any size. These VOPs use
714 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't
715 * go through the buffer cache or allow I/O sizes larger than a
716 * block]. we will eventually want to change this.
717 *
718 * issues to consider:
719 * uvm provides the uvm_aiodesc structure for async i/o management.
720 * there are two tailq's in the uvm. structure... one for pending async
721 * i/o and one for "done" async i/o. to do an async i/o one puts
722 * an aiodesc on the "pending" list (protected by splbio()), starts the
723 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect
724 * some sort of "i/o done" function to be called (at splbio(), interrupt
725 * time). this function should remove the aiodesc from the pending list
726 * and place it on the "done" list and wakeup the daemon. the daemon
727 * will run at normal spl() and will remove all items from the "done"
728 * list and call the "aiodone" hook for each done request (see uvm_pager.c).
729 * [in the old vm code, this was done by calling the "put" routine with
730 * null arguments which made the code harder to read and understand because
731 * you had one function ("put") doing two things.]
732 *
733 * so the current pager needs:
734 * int uvn_aiodone(struct uvm_aiodesc *)
735 *
736 * => return 0 (aio finished, free it). otherwise requeue for later collection.
737 * => called with pageq's locked by the daemon.
738 *
739 * general outline:
740 * - "try" to lock object. if fail, just return (will try again later)
741 * - drop "u_nio" (this req is done!)
742 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio }
743 * - get "page" structures (atop?).
744 * - handle "wanted" pages
745 * - handle "released" pages [using pgo_releasepg]
746 * >>> pgo_releasepg may kill the object
747 * dont forget to look at "object" wanted flag in all cases.
748 */
749
750
751 /*
752 * uvn_flush: flush pages out of a uvm object.
753 *
754 * => object should be locked by caller. we may _unlock_ the object
755 * if (and only if) we need to clean a page (PGO_CLEANIT).
756 * we return with the object locked.
757 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller
758 * might want to unlock higher level resources (e.g. vm_map)
759 * before calling flush.
760 * => if PGO_CLEANIT is not set, then we will neither unlock the object
761 * or block.
762 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
763 * for flushing.
764 * => NOTE: we rely on the fact that the object's memq is a TAILQ and
765 * that new pages are inserted on the tail end of the list. thus,
766 * we can make a complete pass through the object in one go by starting
767 * at the head and working towards the tail (new pages are put in
768 * front of us).
769 * => NOTE: we are allowed to lock the page queues, so the caller
770 * must not be holding the lock on them [e.g. pagedaemon had
771 * better not call us with the queues locked]
772 * => we return TRUE unless we encountered some sort of I/O error
773 *
774 * comment on "cleaning" object and PG_BUSY pages:
775 * this routine is holding the lock on the object. the only time
776 * that it can run into a PG_BUSY page that it does not own is if
777 * some other process has started I/O on the page (e.g. either
778 * a pagein, or a pageout). if the PG_BUSY page is being paged
779 * in, then it can not be dirty (!PG_CLEAN) because no one has
780 * had a chance to modify it yet. if the PG_BUSY page is being
781 * paged out then it means that someone else has already started
782 * cleaning the page for us (how nice!). in this case, if we
783 * have syncio specified, then after we make our pass through the
784 * object we need to wait for the other PG_BUSY pages to clear
785 * off (i.e. we need to do an iosync). also note that once a
786 * page is PG_BUSY it must stay in its object until it is un-busyed.
787 *
788 * note on page traversal:
789 * we can traverse the pages in an object either by going down the
790 * linked list in "uobj->memq", or we can go over the address range
791 * by page doing hash table lookups for each address. depending
792 * on how many pages are in the object it may be cheaper to do one
793 * or the other. we set "by_list" to true if we are using memq.
794 * if the cost of a hash lookup was equal to the cost of the list
795 * traversal we could compare the number of pages in the start->stop
796 * range to the total number of pages in the object. however, it
797 * seems that a hash table lookup is more expensive than the linked
798 * list traversal, so we multiply the number of pages in the
799 * start->stop range by a penalty which we define below.
800 */
801
802 #define UVN_HASH_PENALTY 4 /* XXX: a guess */
803
804 boolean_t
805 uvn_flush(uobj, start, stop, flags)
806 struct uvm_object *uobj;
807 voff_t start, stop;
808 int flags;
809 {
810 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
811 struct vm_page *pp, *ppnext, *ptmp;
812 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp;
813 int npages, result, lcv;
814 boolean_t retval, need_iosync, by_list, needs_clean, all;
815 voff_t curoff;
816 u_short pp_version;
817 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist);
818
819 curoff = 0; /* XXX: shut up gcc */
820 /*
821 * get init vals and determine how we are going to traverse object
822 */
823
824 need_iosync = FALSE;
825 retval = TRUE; /* return value */
826 if (flags & PGO_ALLPAGES) {
827 all = TRUE;
828 by_list = TRUE; /* always go by the list */
829 } else {
830 start = trunc_page(start);
831 stop = round_page(stop);
832 #ifdef DEBUG
833 if (stop > round_page(uvn->u_size))
834 printf("uvn_flush: strange, got an out of range "
835 "flush (fixed)\n");
836 #endif
837 all = FALSE;
838 by_list = (uobj->uo_npages <=
839 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY);
840 }
841
842 UVMHIST_LOG(maphist,
843 " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
844 (u_long)start, (u_long)stop, by_list, flags);
845
846 /*
847 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as
848 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint
849 * is wrong it will only prevent us from clustering... it won't break
850 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster
851 * will set them as it syncs PG_CLEAN. This is only an issue if we
852 * are looking at non-inactive pages (because inactive page's PG_CLEAN
853 * bit is always up to date since there are no mappings).
854 * [borrowed PG_CLEANCHK idea from FreeBSD VM]
855 */
856
857 if ((flags & PGO_CLEANIT) != 0 &&
858 uobj->pgops->pgo_mk_pcluster != NULL) {
859 if (by_list) {
860 TAILQ_FOREACH(pp, &uobj->memq, listq) {
861 if (!all &&
862 (pp->offset < start || pp->offset >= stop))
863 continue;
864 atomic_clearbits_int(&pp->pg_flags,
865 PG_CLEANCHK);
866 }
867
868 } else { /* by hash */
869 for (curoff = start ; curoff < stop;
870 curoff += PAGE_SIZE) {
871 pp = uvm_pagelookup(uobj, curoff);
872 if (pp)
873 atomic_clearbits_int(&pp->pg_flags,
874 PG_CLEANCHK);
875 }
876 }
877 }
878
879 /*
880 * now do it. note: we must update ppnext in body of loop or we
881 * will get stuck. we need to use ppnext because we may free "pp"
882 * before doing the next loop.
883 */
884
885 if (by_list) {
886 pp = TAILQ_FIRST(&uobj->memq);
887 } else {
888 curoff = start;
889 pp = uvm_pagelookup(uobj, curoff);
890 }
891
892 ppnext = NULL; /* XXX: shut up gcc */
893 ppsp = NULL; /* XXX: shut up gcc */
894 uvm_lock_pageq(); /* page queues locked */
895
896 /* locked: both page queues and uobj */
897 for ( ; (by_list && pp != NULL) ||
898 (!by_list && curoff < stop) ; pp = ppnext) {
899
900 if (by_list) {
901
902 /*
903 * range check
904 */
905
906 if (!all &&
907 (pp->offset < start || pp->offset >= stop)) {
908 ppnext = TAILQ_NEXT(pp, listq);
909 continue;
910 }
911
912 } else {
913
914 /*
915 * null check
916 */
917
918 curoff += PAGE_SIZE;
919 if (pp == NULL) {
920 if (curoff < stop)
921 ppnext = uvm_pagelookup(uobj, curoff);
922 continue;
923 }
924
925 }
926
927 /*
928 * handle case where we do not need to clean page (either
929 * because we are not clean or because page is not dirty or
930 * is busy):
931 *
932 * NOTE: we are allowed to deactivate a non-wired active
933 * PG_BUSY page, but once a PG_BUSY page is on the inactive
934 * queue it must stay put until it is !PG_BUSY (so as not to
935 * confuse pagedaemon).
936 */
937
938 if ((flags & PGO_CLEANIT) == 0 || (pp->pg_flags & PG_BUSY) != 0) {
939 needs_clean = FALSE;
940 if ((pp->pg_flags & PG_BUSY) != 0 &&
941 (flags & (PGO_CLEANIT|PGO_SYNCIO)) ==
942 (PGO_CLEANIT|PGO_SYNCIO))
943 need_iosync = TRUE;
944 } else {
945 /*
946 * freeing: nuke all mappings so we can sync
947 * PG_CLEAN bit with no race
948 */
949 if ((pp->pg_flags & PG_CLEAN) != 0 &&
950 (flags & PGO_FREE) != 0 &&
951 (pp->pg_flags & PQ_ACTIVE) != 0)
952 pmap_page_protect(pp, VM_PROT_NONE);
953 if ((pp->pg_flags & PG_CLEAN) != 0 &&
954 pmap_is_modified(pp))
955 atomic_clearbits_int(&pp->pg_flags, PG_CLEAN);
956 atomic_setbits_int(&pp->pg_flags, PG_CLEANCHK);
957
958 needs_clean = ((pp->pg_flags & PG_CLEAN) == 0);
959 }
960
961 /*
962 * if we don't need a clean... load ppnext and dispose of pp
963 */
964 if (!needs_clean) {
965 /* load ppnext */
966 if (by_list)
967 ppnext = TAILQ_NEXT(pp, listq);
968 else {
969 if (curoff < stop)
970 ppnext = uvm_pagelookup(uobj, curoff);
971 }
972
973 /* now dispose of pp */
974 if (flags & PGO_DEACTIVATE) {
975 if ((pp->pg_flags & PQ_INACTIVE) == 0 &&
976 pp->wire_count == 0) {
977 pmap_page_protect(pp, VM_PROT_NONE);
978 uvm_pagedeactivate(pp);
979 }
980
981 } else if (flags & PGO_FREE) {
982 if (pp->pg_flags & PG_BUSY) {
983 /* release busy pages */
984 atomic_setbits_int(&pp->pg_flags,
985 PG_RELEASED);
986 } else {
987 pmap_page_protect(pp, VM_PROT_NONE);
988 /* removed page from object */
989 uvm_pagefree(pp);
990 }
991 }
992 /* ppnext is valid so we can continue... */
993 continue;
994 }
995
996 /*
997 * pp points to a page in the locked object that we are
998 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked
999 * for cleaning (PGO_CLEANIT). we clean it now.
1000 *
1001 * let uvm_pager_put attempted a clustered page out.
1002 * note: locked: uobj and page queues.
1003 */
1004
1005 atomic_setbits_int(&pp->pg_flags, PG_BUSY);
1006 UVM_PAGE_OWN(pp, "uvn_flush");
1007 pmap_page_protect(pp, VM_PROT_READ);
1008 pp_version = pp->pg_version;
1009 ReTry:
1010 ppsp = pps;
1011 npages = sizeof(pps) / sizeof(struct vm_page *);
1012
1013 /* locked: page queues, uobj */
1014 result = uvm_pager_put(uobj, pp, &ppsp, &npages,
1015 flags | PGO_DOACTCLUST, start, stop);
1016 /* unlocked: page queues, uobj */
1017
1018 /*
1019 * at this point nothing is locked. if we did an async I/O
1020 * it is remotely possible for the async i/o to complete and
1021 * the page "pp" be freed or what not before we get a chance
1022 * to relock the object. in order to detect this, we have
1023 * saved the version number of the page in "pp_version".
1024 */
1025
1026 /* relock! */
1027 simple_lock(&uobj->vmobjlock);
1028 uvm_lock_pageq();
1029
1030 /*
1031 * VM_PAGER_AGAIN: given the structure of this pager, this
1032 * can only happen when we are doing async I/O and can't
1033 * map the pages into kernel memory (pager_map) due to lack
1034 * of vm space. if this happens we drop back to sync I/O.
1035 */
1036
1037 if (result == VM_PAGER_AGAIN) {
1038 /*
1039 * it is unlikely, but page could have been released
1040 * while we had the object lock dropped. we ignore
1041 * this now and retry the I/O. we will detect and
1042 * handle the released page after the syncio I/O
1043 * completes.
1044 */
1045 #ifdef DIAGNOSTIC
1046 if (flags & PGO_SYNCIO)
1047 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)");
1048 #endif
1049 flags |= PGO_SYNCIO;
1050 goto ReTry;
1051 }
1052
1053 /*
1054 * the cleaning operation is now done. finish up. note that
1055 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us.
1056 * if success (OK, PEND) then uvm_pager_put returns the cluster
1057 * to us in ppsp/npages.
1058 */
1059
1060 /*
1061 * for pending async i/o if we are not deactivating/freeing
1062 * we can move on to the next page.
1063 */
1064
1065 if (result == VM_PAGER_PEND) {
1066
1067 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1068 /*
1069 * no per-page ops: refresh ppnext and continue
1070 */
1071 if (by_list) {
1072 if (pp->pg_version == pp_version)
1073 ppnext = TAILQ_NEXT(pp, listq);
1074 else
1075 /* reset */
1076 ppnext = TAILQ_FIRST(&uobj->memq);
1077 } else {
1078 if (curoff < stop)
1079 ppnext = uvm_pagelookup(uobj,
1080 curoff);
1081 }
1082 continue;
1083 }
1084
1085 /* need to do anything here? */
1086 }
1087
1088 /*
1089 * need to look at each page of the I/O operation. we defer
1090 * processing "pp" until the last trip through this "for" loop
1091 * so that we can load "ppnext" for the main loop after we
1092 * play with the cluster pages [thus the "npages + 1" in the
1093 * loop below].
1094 */
1095
1096 for (lcv = 0 ; lcv < npages + 1 ; lcv++) {
1097
1098 /*
1099 * handle ppnext for outside loop, and saving pp
1100 * until the end.
1101 */
1102 if (lcv < npages) {
1103 if (ppsp[lcv] == pp)
1104 continue; /* skip pp until the end */
1105 ptmp = ppsp[lcv];
1106 } else {
1107 ptmp = pp;
1108
1109 /* set up next page for outer loop */
1110 if (by_list) {
1111 if (pp->pg_version == pp_version)
1112 ppnext = TAILQ_NEXT(pp, listq);
1113 else
1114 /* reset */
1115 ppnext = TAILQ_FIRST(&uobj->memq);
1116 } else {
1117 if (curoff < stop)
1118 ppnext = uvm_pagelookup(uobj, curoff);
1119 }
1120 }
1121
1122 /*
1123 * verify the page didn't get moved while obj was
1124 * unlocked
1125 */
1126 if (result == VM_PAGER_PEND && ptmp->uobject != uobj)
1127 continue;
1128
1129 /*
1130 * unbusy the page if I/O is done. note that for
1131 * pending I/O it is possible that the I/O op
1132 * finished before we relocked the object (in
1133 * which case the page is no longer busy).
1134 */
1135
1136 if (result != VM_PAGER_PEND) {
1137 if (ptmp->pg_flags & PG_WANTED)
1138 /* still holding object lock */
1139 wakeup(ptmp);
1140
1141 atomic_clearbits_int(&ptmp->pg_flags,
1142 PG_WANTED|PG_BUSY);
1143 UVM_PAGE_OWN(ptmp, NULL);
1144 if (ptmp->pg_flags & PG_RELEASED) {
1145
1146 /* pgo_releasepg wants this */
1147 uvm_unlock_pageq();
1148 if (!uvn_releasepg(ptmp, NULL))
1149 return (TRUE);
1150
1151 uvm_lock_pageq(); /* relock */
1152 continue; /* next page */
1153
1154 } else {
1155 atomic_setbits_int(&ptmp->pg_flags,
1156 PG_CLEAN|PG_CLEANCHK);
1157 if ((flags & PGO_FREE) == 0)
1158 pmap_clear_modify(ptmp);
1159 }
1160 }
1161
1162 /*
1163 * dispose of page
1164 */
1165
1166 if (flags & PGO_DEACTIVATE) {
1167 if ((pp->pg_flags & PQ_INACTIVE) == 0 &&
1168 pp->wire_count == 0) {
1169 pmap_page_protect(ptmp, VM_PROT_NONE);
1170 uvm_pagedeactivate(ptmp);
1171 }
1172
1173 } else if (flags & PGO_FREE) {
1174 if (result == VM_PAGER_PEND) {
1175 if ((ptmp->pg_flags & PG_BUSY) != 0)
1176 /* signal for i/o done */
1177 atomic_setbits_int(
1178 &ptmp->pg_flags,
1179 PG_RELEASED);
1180 } else {
1181 if (result != VM_PAGER_OK) {
1182 printf("uvn_flush: obj=%p, "
1183 "offset=0x%llx. error "
1184 "during pageout.\n",
1185 pp->uobject,
1186 (long long)pp->offset);
1187 printf("uvn_flush: WARNING: "
1188 "changes to page may be "
1189 "lost!\n");
1190 retval = FALSE;
1191 }
1192 pmap_page_protect(ptmp, VM_PROT_NONE);
1193 uvm_pagefree(ptmp);
1194 }
1195 }
1196
1197 } /* end of "lcv" for loop */
1198
1199 } /* end of "pp" for loop */
1200
1201 /*
1202 * done with pagequeues: unlock
1203 */
1204 uvm_unlock_pageq();
1205
1206 /*
1207 * now wait for all I/O if required.
1208 */
1209 if (need_iosync) {
1210
1211 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0);
1212 while (uvn->u_nio != 0) {
1213 uvn->u_flags |= UVM_VNODE_IOSYNC;
1214 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock,
1215 FALSE, "uvn_flush",0);
1216 simple_lock(&uvn->u_obj.vmobjlock);
1217 }
1218 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED)
1219 wakeup(&uvn->u_flags);
1220 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED);
1221 }
1222
1223 /* return, with object locked! */
1224 UVMHIST_LOG(maphist,"<- done (retval=0x%lx)",retval,0,0,0);
1225 return(retval);
1226 }
1227
1228 /*
1229 * uvn_cluster
1230 *
1231 * we are about to do I/O in an object at offset. this function is called
1232 * to establish a range of offsets around "offset" in which we can cluster
1233 * I/O.
1234 *
1235 * - currently doesn't matter if obj locked or not.
1236 */
1237
1238 void
1239 uvn_cluster(uobj, offset, loffset, hoffset)
1240 struct uvm_object *uobj;
1241 voff_t offset;
1242 voff_t *loffset, *hoffset; /* OUT */
1243 {
1244 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
1245 *loffset = offset;
1246
1247 if (*loffset >= uvn->u_size)
1248 panic("uvn_cluster: offset out of range");
1249
1250 /*
1251 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value.
1252 */
1253 *hoffset = *loffset + MAXBSIZE;
1254 if (*hoffset > round_page(uvn->u_size)) /* past end? */
1255 *hoffset = round_page(uvn->u_size);
1256
1257 return;
1258 }
1259
1260 /*
1261 * uvn_put: flush page data to backing store.
1262 *
1263 * => prefer map unlocked (not required)
1264 * => object must be locked! we will _unlock_ it before starting I/O.
1265 * => flags: PGO_SYNCIO -- use sync. I/O
1266 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed)
1267 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync.
1268 * [thus we never do async i/o! see iodone comment]
1269 */
1270
1271 int
1272 uvn_put(uobj, pps, npages, flags)
1273 struct uvm_object *uobj;
1274 struct vm_page **pps;
1275 int npages, flags;
1276 {
1277 int retval;
1278
1279 /* note: object locked */
1280 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE);
1281 /* note: object unlocked */
1282
1283 return(retval);
1284 }
1285
1286
1287 /*
1288 * uvn_get: get pages (synchronously) from backing store
1289 *
1290 * => prefer map unlocked (not required)
1291 * => object must be locked! we will _unlock_ it before starting any I/O.
1292 * => flags: PGO_ALLPAGES: get all of the pages
1293 * PGO_LOCKED: fault data structures are locked
1294 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
1295 * => NOTE: caller must check for released pages!!
1296 */
1297
1298 int
1299 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
1300 struct uvm_object *uobj;
1301 voff_t offset;
1302 struct vm_page **pps; /* IN/OUT */
1303 int *npagesp; /* IN (OUT if PGO_LOCKED) */
1304 int centeridx, advice, flags;
1305 vm_prot_t access_type;
1306 {
1307 voff_t current_offset;
1308 struct vm_page *ptmp;
1309 int lcv, result, gotpages;
1310 boolean_t done;
1311 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist);
1312 UVMHIST_LOG(maphist, "flags=%ld", flags,0,0,0);
1313
1314 /*
1315 * step 1: handled the case where fault data structures are locked.
1316 */
1317
1318 if (flags & PGO_LOCKED) {
1319
1320 /*
1321 * gotpages is the current number of pages we've gotten (which
1322 * we pass back up to caller via *npagesp.
1323 */
1324
1325 gotpages = 0;
1326
1327 /*
1328 * step 1a: get pages that are already resident. only do this
1329 * if the data structures are locked (i.e. the first time
1330 * through).
1331 */
1332
1333 done = TRUE; /* be optimistic */
1334
1335 for (lcv = 0, current_offset = offset ; lcv < *npagesp ;
1336 lcv++, current_offset += PAGE_SIZE) {
1337
1338 /* do we care about this page? if not, skip it */
1339 if (pps[lcv] == PGO_DONTCARE)
1340 continue;
1341
1342 /* lookup page */
1343 ptmp = uvm_pagelookup(uobj, current_offset);
1344
1345 /* to be useful must get a non-busy, non-released pg */
1346 if (ptmp == NULL ||
1347 (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
1348 if (lcv == centeridx || (flags & PGO_ALLPAGES)
1349 != 0)
1350 done = FALSE; /* need to do a wait or I/O! */
1351 continue;
1352 }
1353
1354 /*
1355 * useful page: busy/lock it and plug it in our
1356 * result array
1357 */
1358 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY);
1359 UVM_PAGE_OWN(ptmp, "uvn_get1");
1360 pps[lcv] = ptmp;
1361 gotpages++;
1362
1363 } /* "for" lcv loop */
1364
1365 /*
1366 * XXX: given the "advice", should we consider async read-ahead?
1367 * XXX: fault current does deactive of pages behind us. is
1368 * this good (other callers might now).
1369 */
1370 /*
1371 * XXX: read-ahead currently handled by buffer cache (bread)
1372 * level.
1373 * XXX: no async i/o available.
1374 * XXX: so we don't do anything now.
1375 */
1376
1377 /*
1378 * step 1c: now we've either done everything needed or we to
1379 * unlock and do some waiting or I/O.
1380 */
1381
1382 *npagesp = gotpages; /* let caller know */
1383 if (done)
1384 return(VM_PAGER_OK); /* bingo! */
1385 else
1386 /* EEK! Need to unlock and I/O */
1387 return(VM_PAGER_UNLOCK);
1388 }
1389
1390 /*
1391 * step 2: get non-resident or busy pages.
1392 * object is locked. data structures are unlocked.
1393 *
1394 * XXX: because we can't do async I/O at this level we get things
1395 * page at a time (otherwise we'd chunk). the VOP_READ() will do
1396 * async-read-ahead for us at a lower level.
1397 */
1398
1399 for (lcv = 0, current_offset = offset;
1400 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) {
1401
1402 /* skip over pages we've already gotten or don't want */
1403 /* skip over pages we don't _have_ to get */
1404 if (pps[lcv] != NULL || (lcv != centeridx &&
1405 (flags & PGO_ALLPAGES) == 0))
1406 continue;
1407
1408 /*
1409 * we have yet to locate the current page (pps[lcv]). we first
1410 * look for a page that is already at the current offset. if
1411 * we fine a page, we check to see if it is busy or released.
1412 * if that is the case, then we sleep on the page until it is
1413 * no longer busy or released and repeat the lookup. if the
1414 * page we found is neither busy nor released, then we busy it
1415 * (so we own it) and plug it into pps[lcv]. this breaks the
1416 * following while loop and indicates we are ready to move on
1417 * to the next page in the "lcv" loop above.
1418 *
1419 * if we exit the while loop with pps[lcv] still set to NULL,
1420 * then it means that we allocated a new busy/fake/clean page
1421 * ptmp in the object and we need to do I/O to fill in the data.
1422 */
1423
1424 while (pps[lcv] == NULL) { /* top of "pps" while loop */
1425
1426 /* look for a current page */
1427 ptmp = uvm_pagelookup(uobj, current_offset);
1428
1429 /* nope? allocate one now (if we can) */
1430 if (ptmp == NULL) {
1431
1432 ptmp = uvm_pagealloc(uobj, current_offset,
1433 NULL, 0);
1434
1435 /* out of RAM? */
1436 if (ptmp == NULL) {
1437 simple_unlock(&uobj->vmobjlock);
1438 uvm_wait("uvn_getpage");
1439 simple_lock(&uobj->vmobjlock);
1440
1441 /* goto top of pps while loop */
1442 continue;
1443 }
1444
1445 /*
1446 * got new page ready for I/O. break pps
1447 * while loop. pps[lcv] is still NULL.
1448 */
1449 break;
1450 }
1451
1452 /* page is there, see if we need to wait on it */
1453 if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
1454 atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
1455 UVM_UNLOCK_AND_WAIT(ptmp,
1456 &uobj->vmobjlock, FALSE, "uvn_get",0);
1457 simple_lock(&uobj->vmobjlock);
1458 continue; /* goto top of pps while loop */
1459 }
1460
1461 /*
1462 * if we get here then the page has become resident
1463 * and unbusy between steps 1 and 2. we busy it
1464 * now (so we own it) and set pps[lcv] (so that we
1465 * exit the while loop).
1466 */
1467 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY);
1468 UVM_PAGE_OWN(ptmp, "uvn_get2");
1469 pps[lcv] = ptmp;
1470 }
1471
1472 /*
1473 * if we own the a valid page at the correct offset, pps[lcv]
1474 * will point to it. nothing more to do except go to the
1475 * next page.
1476 */
1477
1478 if (pps[lcv])
1479 continue; /* next lcv */
1480
1481 /*
1482 * we have a "fake/busy/clean" page that we just allocated. do
1483 * I/O to fill it with valid data. note that object must be
1484 * locked going into uvn_io, but will be unlocked afterwards.
1485 */
1486
1487 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1,
1488 PGO_SYNCIO, UIO_READ);
1489
1490 /*
1491 * I/O done. object is unlocked (by uvn_io). because we used
1492 * syncio the result can not be PEND or AGAIN. we must relock
1493 * and check for errors.
1494 */
1495
1496 /* lock object. check for errors. */
1497 simple_lock(&uobj->vmobjlock);
1498 if (result != VM_PAGER_OK) {
1499 if (ptmp->pg_flags & PG_WANTED)
1500 /* object lock still held */
1501 wakeup(ptmp);
1502
1503 atomic_clearbits_int(&ptmp->pg_flags,
1504 PG_WANTED|PG_BUSY);
1505 UVM_PAGE_OWN(ptmp, NULL);
1506 uvm_lock_pageq();
1507 uvm_pagefree(ptmp);
1508 uvm_unlock_pageq();
1509 simple_unlock(&uobj->vmobjlock);
1510 return(result);
1511 }
1512
1513 /*
1514 * we got the page! clear the fake flag (indicates valid
1515 * data now in page) and plug into our result array. note
1516 * that page is still busy.
1517 *
1518 * it is the callers job to:
1519 * => check if the page is released
1520 * => unbusy the page
1521 * => activate the page
1522 */
1523
1524 /* data is valid ... */
1525 atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE);
1526 pmap_clear_modify(ptmp); /* ... and clean */
1527 pps[lcv] = ptmp;
1528
1529 } /* lcv loop */
1530
1531 /*
1532 * finally, unlock object and return.
1533 */
1534
1535 simple_unlock(&uobj->vmobjlock);
1536 return (VM_PAGER_OK);
1537 }
1538
1539 /*
1540 * uvn_io: do I/O to a vnode
1541 *
1542 * => prefer map unlocked (not required)
1543 * => object must be locked! we will _unlock_ it before starting I/O.
1544 * => flags: PGO_SYNCIO -- use sync. I/O
1545 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync.
1546 * [thus we never do async i/o! see iodone comment]
1547 */
1548
1549 int
1550 uvn_io(uvn, pps, npages, flags, rw)
1551 struct uvm_vnode *uvn;
1552 vm_page_t *pps;
1553 int npages, flags, rw;
1554 {
1555 struct vnode *vn;
1556 struct uio uio;
1557 struct iovec iov;
1558 vaddr_t kva;
1559 off_t file_offset;
1560 int waitf, result, mapinflags;
1561 size_t got, wanted;
1562 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist);
1563
1564 UVMHIST_LOG(maphist, "rw=%ld", rw,0,0,0);
1565
1566 /*
1567 * init values
1568 */
1569
1570 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT;
1571 vn = (struct vnode *) uvn;
1572 file_offset = pps[0]->offset;
1573
1574 /*
1575 * check for sync'ing I/O.
1576 */
1577
1578 while (uvn->u_flags & UVM_VNODE_IOSYNC) {
1579 if (waitf == M_NOWAIT) {
1580 simple_unlock(&uvn->u_obj.vmobjlock);
1581 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0);
1582 return(VM_PAGER_AGAIN);
1583 }
1584 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED;
1585 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock,
1586 FALSE, "uvn_iosync",0);
1587 simple_lock(&uvn->u_obj.vmobjlock);
1588 }
1589
1590 /*
1591 * check size
1592 */
1593
1594 if (file_offset >= uvn->u_size) {
1595 simple_unlock(&uvn->u_obj.vmobjlock);
1596 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0);
1597 return(VM_PAGER_BAD);
1598 }
1599
1600 /*
1601 * first try and map the pages in (without waiting)
1602 */
1603
1604 mapinflags = (rw == UIO_READ) ?
1605 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE;
1606
1607 kva = uvm_pagermapin(pps, npages, mapinflags);
1608 if (kva == 0 && waitf == M_NOWAIT) {
1609 simple_unlock(&uvn->u_obj.vmobjlock);
1610 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0);
1611 return(VM_PAGER_AGAIN);
1612 }
1613
1614 /*
1615 * ok, now bump u_nio up. at this point we are done with uvn
1616 * and can unlock it. if we still don't have a kva, try again
1617 * (this time with sleep ok).
1618 */
1619
1620 uvn->u_nio++; /* we have an I/O in progress! */
1621 simple_unlock(&uvn->u_obj.vmobjlock);
1622 /* NOTE: object now unlocked */
1623 if (kva == 0)
1624 kva = uvm_pagermapin(pps, npages,
1625 mapinflags | UVMPAGER_MAPIN_WAITOK);
1626
1627 /*
1628 * ok, mapped in. our pages are PG_BUSY so they are not going to
1629 * get touched (so we can look at "offset" without having to lock
1630 * the object). set up for I/O.
1631 */
1632
1633 /*
1634 * fill out uio/iov
1635 */
1636
1637 iov.iov_base = (caddr_t) kva;
1638 wanted = npages << PAGE_SHIFT;
1639 if (file_offset + wanted > uvn->u_size)
1640 wanted = uvn->u_size - file_offset; /* XXX: needed? */
1641 iov.iov_len = wanted;
1642 uio.uio_iov = &iov;
1643 uio.uio_iovcnt = 1;
1644 uio.uio_offset = file_offset;
1645 uio.uio_segflg = UIO_SYSSPACE;
1646 uio.uio_rw = rw;
1647 uio.uio_resid = wanted;
1648 uio.uio_procp = curproc;
1649
1650 /*
1651 * do the I/O! (XXX: curproc?)
1652 */
1653
1654 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0);
1655
1656 /*
1657 * This process may already have this vnode locked, if we faulted in
1658 * copyin() or copyout() on a region backed by this vnode
1659 * while doing I/O to the vnode. If this is the case, don't
1660 * panic.. instead, return the error to the user.
1661 *
1662 * XXX this is a stopgap to prevent a panic.
1663 * Ideally, this kind of operation *should* work.
1664 */
1665 result = 0;
1666 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
1667 result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL, curproc);
1668
1669 if (result == 0) {
1670 /* NOTE: vnode now locked! */
1671
1672 if (rw == UIO_READ)
1673 result = VOP_READ(vn, &uio, 0, curproc->p_ucred);
1674 else
1675 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred);
1676
1677 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
1678 VOP_UNLOCK(vn, 0, curproc);
1679 }
1680
1681 /* NOTE: vnode now unlocked (unless vnislocked) */
1682
1683 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0);
1684
1685 /*
1686 * result == unix style errno (0 == OK!)
1687 *
1688 * zero out rest of buffer (if needed)
1689 */
1690
1691 if (result == 0) {
1692 got = wanted - uio.uio_resid;
1693
1694 if (wanted && got == 0) {
1695 result = EIO; /* XXX: error? */
1696 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) {
1697 memset((void *) (kva + got), 0,
1698 (npages << PAGE_SHIFT) - got);
1699 }
1700 }
1701
1702 /*
1703 * now remove pager mapping
1704 */
1705 uvm_pagermapout(kva, npages);
1706
1707 /*
1708 * now clean up the object (i.e. drop I/O count)
1709 */
1710
1711 simple_lock(&uvn->u_obj.vmobjlock);
1712 /* NOTE: object now locked! */
1713
1714 uvn->u_nio--; /* I/O DONE! */
1715 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) {
1716 wakeup(&uvn->u_nio);
1717 }
1718 simple_unlock(&uvn->u_obj.vmobjlock);
1719 /* NOTE: object now unlocked! */
1720
1721 /*
1722 * done!
1723 */
1724
1725 UVMHIST_LOG(maphist, "<- done (result %ld)", result,0,0,0);
1726 if (result == 0)
1727 return(VM_PAGER_OK);
1728 else
1729 return(VM_PAGER_ERROR);
1730 }
1731
1732 /*
1733 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference
1734 * is gone we will kill the object (flushing dirty pages back to the vnode
1735 * if needed).
1736 *
1737 * => returns TRUE if there was no uvm_object attached or if there was
1738 * one and we killed it [i.e. if there is no active uvn]
1739 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if
1740 * needed]
1741 *
1742 * => XXX: given that we now kill uvn's when a vnode is recycled (without
1743 * having to hold a reference on the vnode) and given a working
1744 * uvm_vnp_sync(), how does that effect the need for this function?
1745 * [XXXCDC: seems like it can die?]
1746 *
1747 * => XXX: this function should DIE once we merge the VM and buffer
1748 * cache.
1749 *
1750 * research shows that this is called in the following places:
1751 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode
1752 * changes sizes
1753 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we
1754 * are written to
1755 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit
1756 * is off
1757 * ffs_realloccg: when we can't extend the current block and have
1758 * to allocate a new one we call this [XXX: why?]
1759 * nfsrv_rename, rename_files: called when the target filename is there
1760 * and we want to remove it
1761 * nfsrv_remove, sys_unlink: called on file we are removing
1762 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache
1763 * then return "text busy"
1764 * nfs_open: seems to uncache any file opened with nfs
1765 * vn_writechk: if VTEXT vnode and can't uncache return "text busy"
1766 */
1767
1768 boolean_t
1769 uvm_vnp_uncache(vp)
1770 struct vnode *vp;
1771 {
1772 struct uvm_vnode *uvn = &vp->v_uvm;
1773
1774 /*
1775 * lock uvn part of the vnode and check to see if we need to do anything
1776 */
1777
1778 simple_lock(&uvn->u_obj.vmobjlock);
1779 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 ||
1780 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) {
1781 simple_unlock(&uvn->u_obj.vmobjlock);
1782 return(TRUE);
1783 }
1784
1785 /*
1786 * we have a valid, non-blocked uvn. clear persist flag.
1787 * if uvn is currently active we can return now.
1788 */
1789
1790 uvn->u_flags &= ~UVM_VNODE_CANPERSIST;
1791 if (uvn->u_obj.uo_refs) {
1792 simple_unlock(&uvn->u_obj.vmobjlock);
1793 return(FALSE);
1794 }
1795
1796 /*
1797 * uvn is currently persisting! we have to gain a reference to
1798 * it so that we can call uvn_detach to kill the uvn.
1799 */
1800
1801 VREF(vp); /* seems ok, even with VOP_LOCK */
1802 uvn->u_obj.uo_refs++; /* value is now 1 */
1803 simple_unlock(&uvn->u_obj.vmobjlock);
1804
1805
1806 #ifdef DEBUG
1807 /*
1808 * carry over sanity check from old vnode pager: the vnode should
1809 * be VOP_LOCK'd, and we confirm it here.
1810 */
1811 if (!VOP_ISLOCKED(vp)) {
1812 boolean_t is_ok_anyway = FALSE;
1813 #if defined(NFSCLIENT)
1814 extern int (**nfsv2_vnodeop_p)(void *);
1815 extern int (**spec_nfsv2nodeop_p)(void *);
1816 #if defined(FIFO)
1817 extern int (**fifo_nfsv2nodeop_p)(void *);
1818 #endif /* defined(FIFO) */
1819
1820 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */
1821 if (vp->v_op == nfsv2_vnodeop_p ||
1822 vp->v_op == spec_nfsv2nodeop_p) {
1823 is_ok_anyway = TRUE;
1824 }
1825 #if defined(FIFO)
1826 if (vp->v_op == fifo_nfsv2nodeop_p) {
1827 is_ok_anyway = TRUE;
1828 }
1829 #endif /* defined(FIFO) */
1830 #endif /* defined(NFSSERVER) || defined(NFSCLIENT) */
1831 if (!is_ok_anyway)
1832 panic("uvm_vnp_uncache: vnode not locked!");
1833 }
1834 #endif /* DEBUG */
1835
1836 /*
1837 * now drop our reference to the vnode. if we have the sole
1838 * reference to the vnode then this will cause it to die [as we
1839 * just cleared the persist flag]. we have to unlock the vnode
1840 * while we are doing this as it may trigger I/O.
1841 *
1842 * XXX: it might be possible for uvn to get reclaimed while we are
1843 * unlocked causing us to return TRUE when we should not. we ignore
1844 * this as a false-positive return value doesn't hurt us.
1845 */
1846 VOP_UNLOCK(vp, 0, curproc);
1847 uvn_detach(&uvn->u_obj);
1848 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curproc);
1849
1850 /*
1851 * and return...
1852 */
1853
1854 return(TRUE);
1855 }
1856
1857 /*
1858 * uvm_vnp_setsize: grow or shrink a vnode uvn
1859 *
1860 * grow => just update size value
1861 * shrink => toss un-needed pages
1862 *
1863 * => we assume that the caller has a reference of some sort to the
1864 * vnode in question so that it will not be yanked out from under
1865 * us.
1866 *
1867 * called from:
1868 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos])
1869 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write)
1870 * => ffs_balloc [XXX: why? doesn't WRITE handle?]
1871 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr
1872 * => union fs: union_newsize
1873 */
1874
1875 void
1876 uvm_vnp_setsize(vp, newsize)
1877 struct vnode *vp;
1878 voff_t newsize;
1879 {
1880 struct uvm_vnode *uvn = &vp->v_uvm;
1881
1882 /*
1883 * lock uvn and check for valid object, and if valid: do it!
1884 */
1885 simple_lock(&uvn->u_obj.vmobjlock);
1886 if (uvn->u_flags & UVM_VNODE_VALID) {
1887
1888 /*
1889 * now check if the size has changed: if we shrink we had better
1890 * toss some pages...
1891 */
1892
1893 if (uvn->u_size > newsize) {
1894 (void)uvn_flush(&uvn->u_obj, newsize,
1895 uvn->u_size, PGO_FREE);
1896 }
1897 uvn->u_size = newsize;
1898 }
1899 simple_unlock(&uvn->u_obj.vmobjlock);
1900
1901 /*
1902 * done
1903 */
1904 return;
1905 }
1906
1907 /*
1908 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes.
1909 *
1910 * => called from sys_sync with no VM structures locked
1911 * => only one process can do a sync at a time (because the uvn
1912 * structure only has one queue for sync'ing). we ensure this
1913 * by holding the uvn_sync_lock while the sync is in progress.
1914 * other processes attempting a sync will sleep on this lock
1915 * until we are done.
1916 */
1917
1918 void
1919 uvm_vnp_sync(mp)
1920 struct mount *mp;
1921 {
1922 struct uvm_vnode *uvn;
1923 struct vnode *vp;
1924 boolean_t got_lock;
1925
1926 /*
1927 * step 1: ensure we are only ones using the uvn_sync_q by locking
1928 * our lock...
1929 */
1930 rw_enter_write(&uvn_sync_lock);
1931
1932 /*
1933 * step 2: build up a simpleq of uvns of interest based on the
1934 * write list. we gain a reference to uvns of interest.
1935 */
1936 SIMPLEQ_INIT(&uvn_sync_q);
1937 LIST_FOREACH(uvn, &uvn_wlist, u_wlist) {
1938
1939 vp = (struct vnode *) uvn;
1940 if (mp && vp->v_mount != mp)
1941 continue;
1942
1943 /* attempt to gain reference */
1944 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) ==
1945 FALSE &&
1946 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0)
1947 /* spin */;
1948
1949 /*
1950 * we will exit the loop if either if the following are true:
1951 * - we got the lock [always true if NCPU == 1]
1952 * - we failed to get the lock but noticed the vnode was
1953 * "blocked" -- in this case the vnode must be a dying
1954 * vnode, and since dying vnodes are in the process of
1955 * being flushed out, we can safely skip this one
1956 *
1957 * we want to skip over the vnode if we did not get the lock,
1958 * or if the vnode is already dying (due to the above logic).
1959 *
1960 * note that uvn must already be valid because we found it on
1961 * the wlist (this also means it can't be ALOCK'd).
1962 */
1963 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) {
1964 if (got_lock)
1965 simple_unlock(&uvn->u_obj.vmobjlock);
1966 continue; /* skip it */
1967 }
1968
1969 /*
1970 * gain reference. watch out for persisting uvns (need to
1971 * regain vnode REF).
1972 */
1973 if (uvn->u_obj.uo_refs == 0)
1974 VREF(vp);
1975 uvn->u_obj.uo_refs++;
1976 simple_unlock(&uvn->u_obj.vmobjlock);
1977
1978 /*
1979 * got it!
1980 */
1981 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq);
1982 }
1983
1984 /*
1985 * step 3: we now have a list of uvn's that may need cleaning.
1986 * we are holding the uvn_sync_lock.
1987 */
1988
1989 SIMPLEQ_FOREACH(uvn, &uvn_sync_q, u_syncq) {
1990 simple_lock(&uvn->u_obj.vmobjlock);
1991 #ifdef DEBUG
1992 if (uvn->u_flags & UVM_VNODE_DYING) {
1993 printf("uvm_vnp_sync: dying vnode on sync list\n");
1994 }
1995 #endif
1996 uvn_flush(&uvn->u_obj, 0, 0,
1997 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST);
1998
1999 /*
2000 * if we have the only reference and we just cleaned the uvn,
2001 * then we can pull it out of the UVM_VNODE_WRITEABLE state
2002 * thus allowing us to avoid thinking about flushing it again
2003 * on later sync ops.
2004 */
2005 if (uvn->u_obj.uo_refs == 1 &&
2006 (uvn->u_flags & UVM_VNODE_WRITEABLE)) {
2007 LIST_REMOVE(uvn, u_wlist);
2008 uvn->u_flags &= ~UVM_VNODE_WRITEABLE;
2009 }
2010
2011 simple_unlock(&uvn->u_obj.vmobjlock);
2012
2013 /* now drop our reference to the uvn */
2014 uvn_detach(&uvn->u_obj);
2015 }
2016
2017 /*
2018 * done! release sync lock
2019 */
2020 rw_exit_write(&uvn_sync_lock);
2021 }