1 /* $OpenBSD: uvm_fault.c,v 1.49 2007/06/18 21:51:15 pedro Exp $ */
2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */
3
4 /*
5 *
6 * Copyright (c) 1997 Charles D. Cranor and Washington University.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by Charles D. Cranor and
20 * Washington University.
21 * 4. The name of the author may not be used to endorse or promote products
22 * derived from this software without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 *
35 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
36 */
37
38 /*
39 * uvm_fault.c: fault handler
40 */
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/proc.h>
46 #include <sys/malloc.h>
47 #include <sys/mman.h>
48 #include <sys/user.h>
49
50 #include <uvm/uvm.h>
51
52 /*
53 *
54 * a word on page faults:
55 *
56 * types of page faults we handle:
57 *
58 * CASE 1: upper layer faults CASE 2: lower layer faults
59 *
60 * CASE 1A CASE 1B CASE 2A CASE 2B
61 * read/write1 write>1 read/write +-cow_write/zero
62 * | | | |
63 * +--|--+ +--|--+ +-----+ + | + | +-----+
64 * amap | V | | ----------->new| | | | ^ |
65 * +-----+ +-----+ +-----+ + | + | +--|--+
66 * | | |
67 * +-----+ +-----+ +--|--+ | +--|--+
68 * uobj | d/c | | d/c | | V | +----| |
69 * +-----+ +-----+ +-----+ +-----+
70 *
71 * d/c = don't care
72 *
73 * case [0]: layerless fault
74 * no amap or uobj is present. this is an error.
75 *
76 * case [1]: upper layer fault [anon active]
77 * 1A: [read] or [write with anon->an_ref == 1]
78 * I/O takes place in top level anon and uobj is not touched.
79 * 1B: [write with anon->an_ref > 1]
80 * new anon is alloc'd and data is copied off ["COW"]
81 *
82 * case [2]: lower layer fault [uobj]
83 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
84 * I/O takes place directly in object.
85 * 2B: [write to copy_on_write] or [read on NULL uobj]
86 * data is "promoted" from uobj to a new anon.
87 * if uobj is null, then we zero fill.
88 *
89 * we follow the standard UVM locking protocol ordering:
90 *
91 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
92 * we hold a PG_BUSY page if we unlock for I/O
93 *
94 *
95 * the code is structured as follows:
96 *
97 * - init the "IN" params in the ufi structure
98 * ReFault:
99 * - do lookups [locks maps], check protection, handle needs_copy
100 * - check for case 0 fault (error)
101 * - establish "range" of fault
102 * - if we have an amap lock it and extract the anons
103 * - if sequential advice deactivate pages behind us
104 * - at the same time check pmap for unmapped areas and anon for pages
105 * that we could map in (and do map it if found)
106 * - check object for resident pages that we could map in
107 * - if (case 2) goto Case2
108 * - >>> handle case 1
109 * - ensure source anon is resident in RAM
110 * - if case 1B alloc new anon and copy from source
111 * - map the correct page in
112 * Case2:
113 * - >>> handle case 2
114 * - ensure source page is resident (if uobj)
115 * - if case 2B alloc new anon and copy from source (could be zero
116 * fill if uobj == NULL)
117 * - map the correct page in
118 * - done!
119 *
120 * note on paging:
121 * if we have to do I/O we place a PG_BUSY page in the correct object,
122 * unlock everything, and do the I/O. when I/O is done we must reverify
123 * the state of the world before assuming that our data structures are
124 * valid. [because mappings could change while the map is unlocked]
125 *
126 * alternative 1: unbusy the page in question and restart the page fault
127 * from the top (ReFault). this is easy but does not take advantage
128 * of the information that we already have from our previous lookup,
129 * although it is possible that the "hints" in the vm_map will help here.
130 *
131 * alternative 2: the system already keeps track of a "version" number of
132 * a map. [i.e. every time you write-lock a map (e.g. to change a
133 * mapping) you bump the version number up by one...] so, we can save
134 * the version number of the map before we release the lock and start I/O.
135 * then when I/O is done we can relock and check the version numbers
136 * to see if anything changed. this might save us some over 1 because
137 * we don't have to unbusy the page and may be less compares(?).
138 *
139 * alternative 3: put in backpointers or a way to "hold" part of a map
140 * in place while I/O is in progress. this could be complex to
141 * implement (especially with structures like amap that can be referenced
142 * by multiple map entries, and figuring out what should wait could be
143 * complex as well...).
144 *
145 * given that we are not currently multiprocessor or multithreaded we might
146 * as well choose alternative 2 now. maybe alternative 3 would be useful
147 * in the future. XXX keep in mind for future consideration//rechecking.
148 */
149
150 /*
151 * local data structures
152 */
153
154 struct uvm_advice {
155 int advice;
156 int nback;
157 int nforw;
158 };
159
160 /*
161 * page range array:
162 * note: index in array must match "advice" value
163 * XXX: borrowed numbers from freebsd. do they work well for us?
164 */
165
166 static struct uvm_advice uvmadvice[] = {
167 { MADV_NORMAL, 3, 4 },
168 { MADV_RANDOM, 0, 0 },
169 { MADV_SEQUENTIAL, 8, 7},
170 };
171
172 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */
173
174 /*
175 * private prototypes
176 */
177
178 static void uvmfault_amapcopy(struct uvm_faultinfo *);
179 static __inline void uvmfault_anonflush(struct vm_anon **, int);
180
181 /*
182 * inline functions
183 */
184
185 /*
186 * uvmfault_anonflush: try and deactivate pages in specified anons
187 *
188 * => does not have to deactivate page if it is busy
189 */
190
191 static __inline void
192 uvmfault_anonflush(anons, n)
193 struct vm_anon **anons;
194 int n;
195 {
196 int lcv;
197 struct vm_page *pg;
198
199 for (lcv = 0 ; lcv < n ; lcv++) {
200 if (anons[lcv] == NULL)
201 continue;
202 simple_lock(&anons[lcv]->an_lock);
203 pg = anons[lcv]->an_page;
204 if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) {
205 uvm_lock_pageq();
206 if (pg->wire_count == 0) {
207 #ifdef UBC
208 pmap_clear_reference(pg);
209 #else
210 pmap_page_protect(pg, VM_PROT_NONE);
211 #endif
212 uvm_pagedeactivate(pg);
213 }
214 uvm_unlock_pageq();
215 }
216 simple_unlock(&anons[lcv]->an_lock);
217 }
218 }
219
220 /*
221 * normal functions
222 */
223
224 /*
225 * uvmfault_amapcopy: clear "needs_copy" in a map.
226 *
227 * => called with VM data structures unlocked (usually, see below)
228 * => we get a write lock on the maps and clear needs_copy for a VA
229 * => if we are out of RAM we sleep (waiting for more)
230 */
231
232 static void
233 uvmfault_amapcopy(ufi)
234 struct uvm_faultinfo *ufi;
235 {
236
237 /*
238 * while we haven't done the job
239 */
240
241 while (1) {
242
243 /*
244 * no mapping? give up.
245 */
246
247 if (uvmfault_lookup(ufi, TRUE) == FALSE)
248 return;
249
250 /*
251 * copy if needed.
252 */
253
254 if (UVM_ET_ISNEEDSCOPY(ufi->entry))
255 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE,
256 ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
257
258 /*
259 * didn't work? must be out of RAM. unlock and sleep.
260 */
261
262 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
263 uvmfault_unlockmaps(ufi, TRUE);
264 uvm_wait("fltamapcopy");
265 continue;
266 }
267
268 /*
269 * got it! unlock and return.
270 */
271
272 uvmfault_unlockmaps(ufi, TRUE);
273 return;
274 }
275 /*NOTREACHED*/
276 }
277
278 /*
279 * uvmfault_anonget: get data in an anon into a non-busy, non-released
280 * page in that anon.
281 *
282 * => maps, amap, and anon locked by caller.
283 * => if we fail (result != VM_PAGER_OK) we unlock everything.
284 * => if we are successful, we return with everything still locked.
285 * => we don't move the page on the queues [gets moved later]
286 * => if we allocate a new page [we_own], it gets put on the queues.
287 * either way, the result is that the page is on the queues at return time
288 * => for pages which are on loan from a uvm_object (and thus are not
289 * owned by the anon): if successful, we return with the owning object
290 * locked. the caller must unlock this object when it unlocks everything
291 * else.
292 */
293
294 int
295 uvmfault_anonget(ufi, amap, anon)
296 struct uvm_faultinfo *ufi;
297 struct vm_amap *amap;
298 struct vm_anon *anon;
299 {
300 boolean_t we_own; /* we own anon's page? */
301 boolean_t locked; /* did we relock? */
302 struct vm_page *pg;
303 int result;
304 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
305
306 result = 0; /* XXX shut up gcc */
307 uvmexp.fltanget++;
308 /* bump rusage counters */
309 if (anon->an_page)
310 curproc->p_addr->u_stats.p_ru.ru_minflt++;
311 else
312 curproc->p_addr->u_stats.p_ru.ru_majflt++;
313
314 /*
315 * loop until we get it, or fail.
316 */
317
318 while (1) {
319
320 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
321 pg = anon->an_page;
322
323 /*
324 * if there is a resident page and it is loaned, then anon
325 * may not own it. call out to uvm_anon_lockpage() to ensure
326 * the real owner of the page has been identified and locked.
327 */
328
329 if (pg && pg->loan_count)
330 pg = uvm_anon_lockloanpg(anon);
331
332 /*
333 * page there? make sure it is not busy/released.
334 */
335
336 if (pg) {
337
338 /*
339 * at this point, if the page has a uobject [meaning
340 * we have it on loan], then that uobject is locked
341 * by us! if the page is busy, we drop all the
342 * locks (including uobject) and try again.
343 */
344
345 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
346 UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
347 return (VM_PAGER_OK);
348 }
349 atomic_setbits_int(&pg->pg_flags, PG_WANTED);
350 uvmexp.fltpgwait++;
351
352 /*
353 * the last unlock must be an atomic unlock+wait on
354 * the owner of page
355 */
356 if (pg->uobject) { /* owner is uobject ? */
357 uvmfault_unlockall(ufi, amap, NULL, anon);
358 UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
359 0,0,0);
360 UVM_UNLOCK_AND_WAIT(pg,
361 &pg->uobject->vmobjlock,
362 FALSE, "anonget1",0);
363 } else {
364 /* anon owns page */
365 uvmfault_unlockall(ufi, amap, NULL, NULL);
366 UVMHIST_LOG(maphist, " unlock+wait on anon",0,
367 0,0,0);
368 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
369 "anonget2",0);
370 }
371 /* ready to relock and try again */
372
373 } else {
374
375 /*
376 * no page, we must try and bring it in.
377 */
378 pg = uvm_pagealloc(NULL, 0, anon, 0);
379
380 if (pg == NULL) { /* out of RAM. */
381
382 uvmfault_unlockall(ufi, amap, NULL, anon);
383 uvmexp.fltnoram++;
384 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
385 0,0,0);
386 uvm_wait("flt_noram1");
387 /* ready to relock and try again */
388
389 } else {
390
391 /* we set the PG_BUSY bit */
392 we_own = TRUE;
393 uvmfault_unlockall(ufi, amap, NULL, anon);
394
395 /*
396 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
397 * page into the uvm_swap_get function with
398 * all data structures unlocked. note that
399 * it is ok to read an_swslot here because
400 * we hold PG_BUSY on the page.
401 */
402 uvmexp.pageins++;
403 result = uvm_swap_get(pg, anon->an_swslot,
404 PGO_SYNCIO);
405
406 /*
407 * we clean up after the i/o below in the
408 * "we_own" case
409 */
410 /* ready to relock and try again */
411 }
412 }
413
414 /*
415 * now relock and try again
416 */
417
418 locked = uvmfault_relock(ufi);
419 if (locked || we_own)
420 simple_lock(&anon->an_lock);
421
422 /*
423 * if we own the page (i.e. we set PG_BUSY), then we need
424 * to clean up after the I/O. there are three cases to
425 * consider:
426 * [1] page released during I/O: free anon and ReFault.
427 * [2] I/O not OK. free the page and cause the fault
428 * to fail.
429 * [3] I/O OK! activate the page and sync with the
430 * non-we_own case (i.e. drop anon lock if not locked).
431 */
432
433 if (we_own) {
434
435 if (pg->pg_flags & PG_WANTED) {
436 /* still holding object lock */
437 wakeup(pg);
438 }
439 /* un-busy! */
440 atomic_clearbits_int(&pg->pg_flags,
441 PG_WANTED|PG_BUSY|PG_FAKE);
442 UVM_PAGE_OWN(pg, NULL);
443
444 /*
445 * if we were RELEASED during I/O, then our anon is
446 * no longer part of an amap. we need to free the
447 * anon and try again.
448 */
449 if (pg->pg_flags & PG_RELEASED) {
450 pmap_page_protect(pg, VM_PROT_NONE);
451 simple_unlock(&anon->an_lock);
452 uvm_anfree(anon); /* frees page for us */
453 if (locked)
454 uvmfault_unlockall(ufi, amap, NULL,
455 NULL);
456 uvmexp.fltpgrele++;
457 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
458 return (VM_PAGER_REFAULT); /* refault! */
459 }
460
461 if (result != VM_PAGER_OK) {
462 KASSERT(result != VM_PAGER_PEND);
463
464 /* remove page from anon */
465 anon->an_page = NULL;
466
467 /*
468 * remove the swap slot from the anon
469 * and mark the anon as having no real slot.
470 * don't free the swap slot, thus preventing
471 * it from being used again.
472 */
473 uvm_swap_markbad(anon->an_swslot, 1);
474 anon->an_swslot = SWSLOT_BAD;
475
476 /*
477 * note: page was never !PG_BUSY, so it
478 * can't be mapped and thus no need to
479 * pmap_page_protect it...
480 */
481 uvm_lock_pageq();
482 uvm_pagefree(pg);
483 uvm_unlock_pageq();
484
485 if (locked)
486 uvmfault_unlockall(ufi, amap, NULL,
487 anon);
488 else
489 simple_unlock(&anon->an_lock);
490 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
491 return (VM_PAGER_ERROR);
492 }
493
494 /*
495 * must be OK, clear modify (already PG_CLEAN)
496 * and activate
497 */
498 pmap_clear_modify(pg);
499 uvm_lock_pageq();
500 uvm_pageactivate(pg);
501 uvm_unlock_pageq();
502 if (!locked)
503 simple_unlock(&anon->an_lock);
504 }
505
506 /*
507 * we were not able to relock. restart fault.
508 */
509
510 if (!locked) {
511 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
512 return (VM_PAGER_REFAULT);
513 }
514
515 /*
516 * verify no one has touched the amap and moved the anon on us.
517 */
518
519 if (ufi != NULL &&
520 amap_lookup(&ufi->entry->aref,
521 ufi->orig_rvaddr - ufi->entry->start) != anon) {
522
523 uvmfault_unlockall(ufi, amap, NULL, anon);
524 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
525 return (VM_PAGER_REFAULT);
526 }
527
528 /*
529 * try it again!
530 */
531
532 uvmexp.fltanretry++;
533 continue;
534
535 } /* while (1) */
536
537 /*NOTREACHED*/
538 }
539
540 /*
541 * F A U L T - m a i n e n t r y p o i n t
542 */
543
544 /*
545 * uvm_fault: page fault handler
546 *
547 * => called from MD code to resolve a page fault
548 * => VM data structures usually should be unlocked. however, it is
549 * possible to call here with the main map locked if the caller
550 * gets a write lock, sets it recursive, and then calls us (c.f.
551 * uvm_map_pageable). this should be avoided because it keeps
552 * the map locked off during I/O.
553 */
554
555 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
556 ~VM_PROT_WRITE : VM_PROT_ALL)
557
558 int
559 uvm_fault(orig_map, vaddr, fault_type, access_type)
560 vm_map_t orig_map;
561 vaddr_t vaddr;
562 vm_fault_t fault_type;
563 vm_prot_t access_type;
564 {
565 struct uvm_faultinfo ufi;
566 vm_prot_t enter_prot;
567 boolean_t wired, narrow, promote, locked, shadowed;
568 int npages, nback, nforw, centeridx, result, lcv, gotpages;
569 vaddr_t startva, currva;
570 voff_t uoff;
571 paddr_t pa;
572 struct vm_amap *amap;
573 struct uvm_object *uobj;
574 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
575 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
576 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
577
578 UVMHIST_LOG(maphist, "(map=%p, vaddr=0x%lx, ft=%ld, at=%ld)",
579 orig_map, vaddr, fault_type, access_type);
580
581 anon = NULL;
582 pg = NULL;
583
584 uvmexp.faults++; /* XXX: locking? */
585
586 /*
587 * init the IN parameters in the ufi
588 */
589
590 ufi.orig_map = orig_map;
591 ufi.orig_rvaddr = trunc_page(vaddr);
592 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
593 if (fault_type == VM_FAULT_WIRE)
594 narrow = TRUE; /* don't look for neighborhood
595 * pages on wire */
596 else
597 narrow = FALSE; /* normal fault */
598
599 /*
600 * "goto ReFault" means restart the page fault from ground zero.
601 */
602 ReFault:
603
604 /*
605 * lookup and lock the maps
606 */
607
608 if (uvmfault_lookup(&ufi, FALSE) == FALSE) {
609 UVMHIST_LOG(maphist, "<- no mapping @ 0x%lx", vaddr, 0,0,0);
610 return (EFAULT);
611 }
612 /* locked: maps(read) */
613
614 #ifdef DIAGNOSTIC
615 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0)
616 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)",
617 ufi.map, vaddr);
618 #endif
619
620 /*
621 * check protection
622 */
623
624 if ((ufi.entry->protection & access_type) != access_type) {
625 UVMHIST_LOG(maphist,
626 "<- protection failure (prot=0x%lx, access=0x%lx)",
627 ufi.entry->protection, access_type, 0, 0);
628 uvmfault_unlockmaps(&ufi, FALSE);
629 return (EACCES);
630 }
631
632 /*
633 * "enter_prot" is the protection we want to enter the page in at.
634 * for certain pages (e.g. copy-on-write pages) this protection can
635 * be more strict than ufi.entry->protection. "wired" means either
636 * the entry is wired or we are fault-wiring the pg.
637 */
638
639 enter_prot = ufi.entry->protection;
640 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE);
641 if (wired)
642 access_type = enter_prot; /* full access for wired */
643
644 /*
645 * handle "needs_copy" case. if we need to copy the amap we will
646 * have to drop our readlock and relock it with a write lock. (we
647 * need a write lock to change anything in a map entry [e.g.
648 * needs_copy]).
649 */
650
651 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {
652 if ((access_type & VM_PROT_WRITE) ||
653 (ufi.entry->object.uvm_obj == NULL)) {
654 /* need to clear */
655 UVMHIST_LOG(maphist,
656 " need to clear needs_copy and refault",0,0,0,0);
657 uvmfault_unlockmaps(&ufi, FALSE);
658 uvmfault_amapcopy(&ufi);
659 uvmexp.fltamcopy++;
660 goto ReFault;
661
662 } else {
663
664 /*
665 * ensure that we pmap_enter page R/O since
666 * needs_copy is still true
667 */
668 enter_prot &= ~VM_PROT_WRITE;
669
670 }
671 }
672
673 /*
674 * identify the players
675 */
676
677 amap = ufi.entry->aref.ar_amap; /* top layer */
678 uobj = ufi.entry->object.uvm_obj; /* bottom layer */
679
680 /*
681 * check for a case 0 fault. if nothing backing the entry then
682 * error now.
683 */
684
685 if (amap == NULL && uobj == NULL) {
686 uvmfault_unlockmaps(&ufi, FALSE);
687 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
688 return (EFAULT);
689 }
690
691 /*
692 * establish range of interest based on advice from mapper
693 * and then clip to fit map entry. note that we only want
694 * to do this the first time through the fault. if we
695 * ReFault we will disable this by setting "narrow" to true.
696 */
697
698 if (narrow == FALSE) {
699
700 /* wide fault (!narrow) */
701 KASSERT(uvmadvice[ufi.entry->advice].advice ==
702 ufi.entry->advice);
703 nback = min(uvmadvice[ufi.entry->advice].nback,
704 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);
705 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);
706 nforw = min(uvmadvice[ufi.entry->advice].nforw,
707 ((ufi.entry->end - ufi.orig_rvaddr) >>
708 PAGE_SHIFT) - 1);
709 /*
710 * note: "-1" because we don't want to count the
711 * faulting page as forw
712 */
713 npages = nback + nforw + 1;
714 centeridx = nback;
715
716 narrow = TRUE; /* ensure only once per-fault */
717
718 } else {
719
720 /* narrow fault! */
721 nback = nforw = 0;
722 startva = ufi.orig_rvaddr;
723 npages = 1;
724 centeridx = 0;
725
726 }
727
728 /* locked: maps(read) */
729 UVMHIST_LOG(maphist, " narrow=%ld, back=%ld, forw=%ld, startva=0x%lx",
730 narrow, nback, nforw, startva);
731 UVMHIST_LOG(maphist, " entry=%p, amap=%p, obj=%p", ufi.entry,
732 amap, uobj, 0);
733
734 /*
735 * if we've got an amap, lock it and extract current anons.
736 */
737
738 if (amap) {
739 anons = anons_store;
740 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,
741 anons, npages);
742 } else {
743 anons = NULL; /* to be safe */
744 }
745
746 /* locked: maps(read), amap(if there) */
747
748 /*
749 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
750 * now and then forget about them (for the rest of the fault).
751 */
752
753 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) {
754
755 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
756 0,0,0,0);
757 /* flush back-page anons? */
758 if (amap)
759 uvmfault_anonflush(anons, nback);
760
761 /* flush object? */
762 if (uobj) {
763 uoff = (startva - ufi.entry->start) + ufi.entry->offset;
764 simple_lock(&uobj->vmobjlock);
765 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff +
766 (nback << PAGE_SHIFT), PGO_DEACTIVATE);
767 simple_unlock(&uobj->vmobjlock);
768 }
769
770 /* now forget about the backpages */
771 if (amap)
772 anons += nback;
773 startva += (nback << PAGE_SHIFT);
774 npages -= nback;
775 nback = centeridx = 0;
776 }
777
778 /* locked: maps(read), amap(if there) */
779
780 /*
781 * map in the backpages and frontpages we found in the amap in hopes
782 * of preventing future faults. we also init the pages[] array as
783 * we go.
784 */
785
786 currva = startva;
787 shadowed = FALSE;
788 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {
789
790 /*
791 * dont play with VAs that are already mapped
792 * except for center)
793 */
794 if (lcv != centeridx &&
795 pmap_extract(ufi.orig_map->pmap, currva, &pa)) {
796 pages[lcv] = PGO_DONTCARE;
797 continue;
798 }
799
800 /*
801 * unmapped or center page. check if any anon at this level.
802 */
803 if (amap == NULL || anons[lcv] == NULL) {
804 pages[lcv] = NULL;
805 continue;
806 }
807
808 /*
809 * check for present page and map if possible. re-activate it.
810 */
811
812 pages[lcv] = PGO_DONTCARE;
813 if (lcv == centeridx) { /* save center for later! */
814 shadowed = TRUE;
815 continue;
816 }
817 anon = anons[lcv];
818 simple_lock(&anon->an_lock);
819 /* ignore loaned pages */
820 if (anon->an_page && anon->an_page->loan_count == 0 &&
821 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
822 uvm_lock_pageq();
823 uvm_pageactivate(anon->an_page); /* reactivate */
824 uvm_unlock_pageq();
825 UVMHIST_LOG(maphist,
826 " MAPPING: n anon: pm=%p, va=0x%lx, pg=%p",
827 ufi.orig_map->pmap, currva, anon->an_page, 0);
828 uvmexp.fltnamap++;
829
830 /*
831 * Since this isn't the page that's actually faulting,
832 * ignore pmap_enter() failures; it's not critical
833 * that we enter these right now.
834 */
835
836 (void) pmap_enter(ufi.orig_map->pmap, currva,
837 VM_PAGE_TO_PHYS(anon->an_page),
838 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
839 enter_prot,
840 PMAP_CANFAIL |
841 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));
842 }
843 simple_unlock(&anon->an_lock);
844 pmap_update(ufi.orig_map->pmap);
845 }
846
847 /* locked: maps(read), amap(if there) */
848 /* (shadowed == TRUE) if there is an anon at the faulting address */
849 UVMHIST_LOG(maphist, " shadowed=%ld, will_get=%ld", shadowed,
850 (uobj && shadowed == FALSE),0,0);
851
852 /*
853 * note that if we are really short of RAM we could sleep in the above
854 * call to pmap_enter with everything locked. bad?
855 *
856 * XXX Actually, that is bad; pmap_enter() should just fail in that
857 * XXX case. --thorpej
858 */
859
860 /*
861 * if the desired page is not shadowed by the amap and we have a
862 * backing object, then we check to see if the backing object would
863 * prefer to handle the fault itself (rather than letting us do it
864 * with the usual pgo_get hook). the backing object signals this by
865 * providing a pgo_fault routine.
866 */
867
868 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) {
869 simple_lock(&uobj->vmobjlock);
870
871 /* locked: maps(read), amap (if there), uobj */
872 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,
873 centeridx, fault_type, access_type,
874 PGO_LOCKED);
875
876 /* locked: nothing, pgo_fault has unlocked everything */
877
878 if (result == VM_PAGER_OK)
879 return (0); /* pgo_fault did pmap enter */
880 else if (result == VM_PAGER_REFAULT)
881 goto ReFault; /* try again! */
882 else
883 return (EACCES);
884 }
885
886 /*
887 * now, if the desired page is not shadowed by the amap and we have
888 * a backing object that does not have a special fault routine, then
889 * we ask (with pgo_get) the object for resident pages that we care
890 * about and attempt to map them in. we do not let pgo_get block
891 * (PGO_LOCKED).
892 *
893 * ("get" has the option of doing a pmap_enter for us)
894 */
895
896 if (uobj && shadowed == FALSE) {
897 simple_lock(&uobj->vmobjlock);
898
899 /* locked (!shadowed): maps(read), amap (if there), uobj */
900 /*
901 * the following call to pgo_get does _not_ change locking state
902 */
903
904 uvmexp.fltlget++;
905 gotpages = npages;
906 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +
907 (startva - ufi.entry->start),
908 pages, &gotpages, centeridx,
909 access_type & MASK(ufi.entry),
910 ufi.entry->advice, PGO_LOCKED);
911
912 /*
913 * check for pages to map, if we got any
914 */
915
916 uobjpage = NULL;
917
918 if (gotpages) {
919 currva = startva;
920 for (lcv = 0 ; lcv < npages ;
921 lcv++, currva += PAGE_SIZE) {
922
923 if (pages[lcv] == NULL ||
924 pages[lcv] == PGO_DONTCARE)
925 continue;
926
927 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0);
928
929 /*
930 * if center page is resident and not
931 * PG_BUSY|PG_RELEASED then pgo_get
932 * made it PG_BUSY for us and gave
933 * us a handle to it. remember this
934 * page as "uobjpage." (for later use).
935 */
936
937 if (lcv == centeridx) {
938 uobjpage = pages[lcv];
939 UVMHIST_LOG(maphist, " got uobjpage "
940 "(%p) with locked get",
941 uobjpage, 0,0,0);
942 continue;
943 }
944
945 /*
946 * note: calling pgo_get with locked data
947 * structures returns us pages which are
948 * neither busy nor released, so we don't
949 * need to check for this. we can just
950 * directly enter the page (after moving it
951 * to the head of the active queue [useful?]).
952 */
953
954 uvm_lock_pageq();
955 uvm_pageactivate(pages[lcv]); /* reactivate */
956 uvm_unlock_pageq();
957 UVMHIST_LOG(maphist,
958 " MAPPING: n obj: pm=%p, va=0x%lx, pg=%p",
959 ufi.orig_map->pmap, currva, pages[lcv], 0);
960 uvmexp.fltnomap++;
961
962 /*
963 * Since this page isn't the page that's
964 * actually fauling, ignore pmap_enter()
965 * failures; it's not critical that we
966 * enter these right now.
967 */
968
969 (void) pmap_enter(ufi.orig_map->pmap, currva,
970 VM_PAGE_TO_PHYS(pages[lcv]),
971 enter_prot & MASK(ufi.entry),
972 PMAP_CANFAIL |
973 (wired ? PMAP_WIRED : 0));
974
975 /*
976 * NOTE: page can't be PG_WANTED or PG_RELEASED
977 * because we've held the lock the whole time
978 * we've had the handle.
979 */
980
981 atomic_clearbits_int(&pages[lcv]->pg_flags,
982 PG_BUSY);
983 UVM_PAGE_OWN(pages[lcv], NULL);
984 } /* for "lcv" loop */
985 pmap_update(ufi.orig_map->pmap);
986 } /* "gotpages" != 0 */
987 /* note: object still _locked_ */
988 } else {
989 uobjpage = NULL;
990 }
991
992 /* locked (shadowed): maps(read), amap */
993 /* locked (!shadowed): maps(read), amap(if there),
994 uobj(if !null), uobjpage(if !null) */
995
996 /*
997 * note that at this point we are done with any front or back pages.
998 * we are now going to focus on the center page (i.e. the one we've
999 * faulted on). if we have faulted on the top (anon) layer
1000 * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1001 * not touched it yet). if we have faulted on the bottom (uobj)
1002 * layer [i.e. case 2] and the page was both present and available,
1003 * then we've got a pointer to it as "uobjpage" and we've already
1004 * made it BUSY.
1005 */
1006
1007 /*
1008 * there are four possible cases we must address: 1A, 1B, 2A, and 2B
1009 */
1010
1011 /*
1012 * redirect case 2: if we are not shadowed, go to case 2.
1013 */
1014
1015 if (shadowed == FALSE)
1016 goto Case2;
1017
1018 /* locked: maps(read), amap */
1019
1020 /*
1021 * handle case 1: fault on an anon in our amap
1022 */
1023
1024 anon = anons[centeridx];
1025 UVMHIST_LOG(maphist, " case 1 fault: anon=%p", anon, 0,0,0);
1026 simple_lock(&anon->an_lock);
1027
1028 /* locked: maps(read), amap, anon */
1029
1030 /*
1031 * no matter if we have case 1A or case 1B we are going to need to
1032 * have the anon's memory resident. ensure that now.
1033 */
1034
1035 /*
1036 * let uvmfault_anonget do the dirty work.
1037 * if it fails (!OK) it will unlock everything for us.
1038 * if it succeeds, locks are still valid and locked.
1039 * also, if it is OK, then the anon's page is on the queues.
1040 * if the page is on loan from a uvm_object, then anonget will
1041 * lock that object for us if it does not fail.
1042 */
1043
1044 result = uvmfault_anonget(&ufi, amap, anon);
1045 switch (result) {
1046 case VM_PAGER_OK:
1047 break;
1048
1049 case VM_PAGER_REFAULT:
1050 goto ReFault;
1051
1052 case VM_PAGER_ERROR:
1053 /*
1054 * An error occured while trying to bring in the
1055 * page -- this is the only error we return right
1056 * now.
1057 */
1058 return (EACCES); /* XXX */
1059
1060 default:
1061 #ifdef DIAGNOSTIC
1062 panic("uvm_fault: uvmfault_anonget -> %d", result);
1063 #else
1064 return (EACCES);
1065 #endif
1066 }
1067
1068 /*
1069 * uobj is non null if the page is on loan from an object (i.e. uobj)
1070 */
1071
1072 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1073
1074 /* locked: maps(read), amap, anon, uobj(if one) */
1075
1076 /*
1077 * special handling for loaned pages
1078 */
1079
1080 if (anon->an_page->loan_count) {
1081
1082 if ((access_type & VM_PROT_WRITE) == 0) {
1083
1084 /*
1085 * for read faults on loaned pages we just cap the
1086 * protection at read-only.
1087 */
1088
1089 enter_prot = enter_prot & ~VM_PROT_WRITE;
1090
1091 } else {
1092 /*
1093 * note that we can't allow writes into a loaned page!
1094 *
1095 * if we have a write fault on a loaned page in an
1096 * anon then we need to look at the anon's ref count.
1097 * if it is greater than one then we are going to do
1098 * a normal copy-on-write fault into a new anon (this
1099 * is not a problem). however, if the reference count
1100 * is one (a case where we would normally allow a
1101 * write directly to the page) then we need to kill
1102 * the loan before we continue.
1103 */
1104
1105 /* >1 case is already ok */
1106 if (anon->an_ref == 1) {
1107
1108 /* get new un-owned replacement page */
1109 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1110 if (pg == NULL) {
1111 uvmfault_unlockall(&ufi, amap, uobj,
1112 anon);
1113 uvm_wait("flt_noram2");
1114 goto ReFault;
1115 }
1116
1117 /*
1118 * copy data, kill loan, and drop uobj lock
1119 * (if any)
1120 */
1121 /* copy old -> new */
1122 uvm_pagecopy(anon->an_page, pg);
1123
1124 /* force reload */
1125 pmap_page_protect(anon->an_page,
1126 VM_PROT_NONE);
1127 uvm_lock_pageq(); /* KILL loan */
1128 if (uobj)
1129 /* if we were loaning */
1130 anon->an_page->loan_count--;
1131 anon->an_page->uanon = NULL;
1132 /* in case we owned */
1133 atomic_clearbits_int(
1134 &anon->an_page->pg_flags, PQ_ANON);
1135 uvm_pageactivate(pg);
1136 uvm_unlock_pageq();
1137 if (uobj) {
1138 simple_unlock(&uobj->vmobjlock);
1139 uobj = NULL;
1140 }
1141
1142 /* install new page in anon */
1143 anon->an_page = pg;
1144 pg->uanon = anon;
1145 atomic_setbits_int(&pg->pg_flags, PQ_ANON);
1146 atomic_clearbits_int(&pg->pg_flags,
1147 PG_BUSY|PG_FAKE);
1148 UVM_PAGE_OWN(pg, NULL);
1149
1150 /* done! */
1151 } /* ref == 1 */
1152 } /* write fault */
1153 } /* loan count */
1154
1155 /*
1156 * if we are case 1B then we will need to allocate a new blank
1157 * anon to transfer the data into. note that we have a lock
1158 * on anon, so no one can busy or release the page until we are done.
1159 * also note that the ref count can't drop to zero here because
1160 * it is > 1 and we are only dropping one ref.
1161 *
1162 * in the (hopefully very rare) case that we are out of RAM we
1163 * will unlock, wait for more RAM, and refault.
1164 *
1165 * if we are out of anon VM we kill the process (XXX: could wait?).
1166 */
1167
1168 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) {
1169
1170 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1171 uvmexp.flt_acow++;
1172 oanon = anon; /* oanon = old, locked anon */
1173 anon = uvm_analloc();
1174 if (anon) {
1175 pg = uvm_pagealloc(NULL, 0, anon, 0);
1176 }
1177
1178 /* check for out of RAM */
1179 if (anon == NULL || pg == NULL) {
1180 if (anon)
1181 uvm_anfree(anon);
1182 uvmfault_unlockall(&ufi, amap, uobj, oanon);
1183 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1184 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
1185 UVMHIST_LOG(maphist,
1186 "<- failed. out of VM",0,0,0,0);
1187 uvmexp.fltnoanon++;
1188 return (ENOMEM);
1189 }
1190
1191 uvmexp.fltnoram++;
1192 uvm_wait("flt_noram3"); /* out of RAM, wait for more */
1193 goto ReFault;
1194 }
1195
1196 /* got all resources, replace anon with nanon */
1197
1198 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */
1199 /* un-busy! new page */
1200 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE);
1201 UVM_PAGE_OWN(pg, NULL);
1202 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
1203 anon, 1);
1204
1205 /* deref: can not drop to zero here by defn! */
1206 oanon->an_ref--;
1207
1208 /*
1209 * note: oanon still locked. anon is _not_ locked, but we
1210 * have the sole references to in from amap which _is_ locked.
1211 * thus, no one can get at it until we are done with it.
1212 */
1213
1214 } else {
1215
1216 uvmexp.flt_anon++;
1217 oanon = anon; /* old, locked anon is same as anon */
1218 pg = anon->an_page;
1219 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1220 enter_prot = enter_prot & ~VM_PROT_WRITE;
1221
1222 }
1223
1224 /* locked: maps(read), amap, oanon */
1225
1226 /*
1227 * now map the page in ...
1228 * XXX: old fault unlocks object before pmap_enter. this seems
1229 * suspect since some other thread could blast the page out from
1230 * under us between the unlock and the pmap_enter.
1231 */
1232
1233 UVMHIST_LOG(maphist, " MAPPING: anon: pm=%p, va=0x%lx, pg=%p",
1234 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);
1235 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1236 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
1237 != 0) {
1238 /*
1239 * No need to undo what we did; we can simply think of
1240 * this as the pmap throwing away the mapping information.
1241 *
1242 * We do, however, have to go through the ReFault path,
1243 * as the map may change while we're asleep.
1244 */
1245 uvmfault_unlockall(&ufi, amap, uobj, oanon);
1246 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1247 if (uvmexp.swpgonly == uvmexp.swpages) {
1248 UVMHIST_LOG(maphist,
1249 "<- failed. out of VM",0,0,0,0);
1250 /* XXX instrumentation */
1251 return (ENOMEM);
1252 }
1253 /* XXX instrumentation */
1254 uvm_wait("flt_pmfail1");
1255 goto ReFault;
1256 }
1257
1258 /*
1259 * ... update the page queues.
1260 */
1261
1262 uvm_lock_pageq();
1263
1264 if (fault_type == VM_FAULT_WIRE) {
1265 uvm_pagewire(pg);
1266
1267 /*
1268 * since the now-wired page cannot be paged out,
1269 * release its swap resources for others to use.
1270 * since an anon with no swap cannot be PG_CLEAN,
1271 * clear its clean flag now.
1272 */
1273 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
1274 uvm_anon_dropswap(anon);
1275 } else {
1276 /* activate it */
1277 uvm_pageactivate(pg);
1278 }
1279
1280 uvm_unlock_pageq();
1281
1282 /*
1283 * done case 1! finish up by unlocking everything and returning success
1284 */
1285
1286 uvmfault_unlockall(&ufi, amap, uobj, oanon);
1287 pmap_update(ufi.orig_map->pmap);
1288 return (0);
1289
1290
1291 Case2:
1292 /*
1293 * handle case 2: faulting on backing object or zero fill
1294 */
1295
1296 /*
1297 * locked:
1298 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1299 */
1300
1301 /*
1302 * note that uobjpage can not be PGO_DONTCARE at this point. we now
1303 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1304 * have a backing object, check and see if we are going to promote
1305 * the data up to an anon during the fault.
1306 */
1307
1308 if (uobj == NULL) {
1309 uobjpage = PGO_DONTCARE;
1310 promote = TRUE; /* always need anon here */
1311 } else {
1312 KASSERT(uobjpage != PGO_DONTCARE);
1313 promote = (access_type & VM_PROT_WRITE) &&
1314 UVM_ET_ISCOPYONWRITE(ufi.entry);
1315 }
1316 UVMHIST_LOG(maphist, " case 2 fault: promote=%ld, zfill=%ld",
1317 promote, (uobj == NULL), 0,0);
1318
1319 /*
1320 * if uobjpage is not null then we do not need to do I/O to get the
1321 * uobjpage.
1322 *
1323 * if uobjpage is null, then we need to unlock and ask the pager to
1324 * get the data for us. once we have the data, we need to reverify
1325 * the state the world. we are currently not holding any resources.
1326 */
1327
1328 if (uobjpage) {
1329 /* update rusage counters */
1330 curproc->p_addr->u_stats.p_ru.ru_minflt++;
1331 } else {
1332 /* update rusage counters */
1333 curproc->p_addr->u_stats.p_ru.ru_majflt++;
1334
1335 /* locked: maps(read), amap(if there), uobj */
1336 uvmfault_unlockall(&ufi, amap, NULL, NULL);
1337 /* locked: uobj */
1338
1339 uvmexp.fltget++;
1340 gotpages = 1;
1341 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;
1342 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1343 0, access_type & MASK(ufi.entry), ufi.entry->advice,
1344 PGO_SYNCIO);
1345
1346 /* locked: uobjpage(if result OK) */
1347
1348 /*
1349 * recover from I/O
1350 */
1351
1352 if (result != VM_PAGER_OK) {
1353 KASSERT(result != VM_PAGER_PEND);
1354
1355 if (result == VM_PAGER_AGAIN) {
1356 UVMHIST_LOG(maphist,
1357 " pgo_get says TRY AGAIN!",0,0,0,0);
1358 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0);
1359 goto ReFault;
1360 }
1361
1362 UVMHIST_LOG(maphist, "<- pgo_get failed (code %ld)",
1363 result, 0,0,0);
1364 return (EACCES); /* XXX i/o error */
1365 }
1366
1367 /* locked: uobjpage */
1368
1369 /*
1370 * re-verify the state of the world by first trying to relock
1371 * the maps. always relock the object.
1372 */
1373
1374 locked = uvmfault_relock(&ufi);
1375 simple_lock(&uobj->vmobjlock);
1376
1377 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
1378 /* locked(!locked): uobj, uobjpage */
1379
1380 /*
1381 * verify that the page has not be released and re-verify
1382 * that amap slot is still free. if there is a problem,
1383 * we unlock and clean up.
1384 */
1385
1386 if ((uobjpage->pg_flags & PG_RELEASED) != 0 ||
1387 (locked && amap &&
1388 amap_lookup(&ufi.entry->aref,
1389 ufi.orig_rvaddr - ufi.entry->start))) {
1390 if (locked)
1391 uvmfault_unlockall(&ufi, amap, NULL, NULL);
1392 locked = FALSE;
1393 }
1394
1395 /*
1396 * didn't get the lock? release the page and retry.
1397 */
1398
1399 if (locked == FALSE) {
1400
1401 UVMHIST_LOG(maphist,
1402 " wasn't able to relock after fault: retry",
1403 0,0,0,0);
1404 if (uobjpage->pg_flags & PG_WANTED)
1405 /* still holding object lock */
1406 wakeup(uobjpage);
1407
1408 if (uobjpage->pg_flags & PG_RELEASED) {
1409 uvmexp.fltpgrele++;
1410 KASSERT(uobj->pgops->pgo_releasepg != NULL);
1411
1412 /* frees page */
1413 if (uobj->pgops->pgo_releasepg(uobjpage,NULL))
1414 /* unlock if still alive */
1415 simple_unlock(&uobj->vmobjlock);
1416 goto ReFault;
1417 }
1418
1419 uvm_lock_pageq();
1420 /* make sure it is in queues */
1421 uvm_pageactivate(uobjpage);
1422
1423 uvm_unlock_pageq();
1424 atomic_clearbits_int(&uobjpage->pg_flags,
1425 PG_BUSY|PG_WANTED);
1426 UVM_PAGE_OWN(uobjpage, NULL);
1427 simple_unlock(&uobj->vmobjlock);
1428 goto ReFault;
1429
1430 }
1431
1432 /*
1433 * we have the data in uobjpage which is PG_BUSY and
1434 * !PG_RELEASED. we are holding object lock (so the page
1435 * can't be released on us).
1436 */
1437
1438 /* locked: maps(read), amap(if !null), uobj, uobjpage */
1439 }
1440
1441 /*
1442 * locked:
1443 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1444 */
1445
1446 /*
1447 * notes:
1448 * - at this point uobjpage can not be NULL
1449 * - at this point uobjpage can not be PG_RELEASED (since we checked
1450 * for it above)
1451 * - at this point uobjpage could be PG_WANTED (handle later)
1452 */
1453
1454 if (promote == FALSE) {
1455
1456 /*
1457 * we are not promoting. if the mapping is COW ensure that we
1458 * don't give more access than we should (e.g. when doing a read
1459 * fault on a COPYONWRITE mapping we want to map the COW page in
1460 * R/O even though the entry protection could be R/W).
1461 *
1462 * set "pg" to the page we want to map in (uobjpage, usually)
1463 */
1464
1465 uvmexp.flt_obj++;
1466 if (UVM_ET_ISCOPYONWRITE(ufi.entry))
1467 enter_prot &= ~VM_PROT_WRITE;
1468 pg = uobjpage; /* map in the actual object */
1469
1470 /* assert(uobjpage != PGO_DONTCARE) */
1471
1472 /*
1473 * we are faulting directly on the page. be careful
1474 * about writing to loaned pages...
1475 */
1476 if (uobjpage->loan_count) {
1477
1478 if ((access_type & VM_PROT_WRITE) == 0) {
1479 /* read fault: cap the protection at readonly */
1480 /* cap! */
1481 enter_prot = enter_prot & ~VM_PROT_WRITE;
1482 } else {
1483 /* write fault: must break the loan here */
1484
1485 /* alloc new un-owned page */
1486 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1487
1488 if (pg == NULL) {
1489 /*
1490 * drop ownership of page, it can't
1491 * be released
1492 */
1493 if (uobjpage->pg_flags & PG_WANTED)
1494 wakeup(uobjpage);
1495 atomic_clearbits_int(
1496 &uobjpage->pg_flags,
1497 PG_BUSY|PG_WANTED);
1498 UVM_PAGE_OWN(uobjpage, NULL);
1499
1500 uvm_lock_pageq();
1501 /* activate: we will need it later */
1502 uvm_pageactivate(uobjpage);
1503
1504 uvm_unlock_pageq();
1505 uvmfault_unlockall(&ufi, amap, uobj,
1506 NULL);
1507 UVMHIST_LOG(maphist,
1508 " out of RAM breaking loan, waiting",
1509 0,0,0,0);
1510 uvmexp.fltnoram++;
1511 uvm_wait("flt_noram4");
1512 goto ReFault;
1513 }
1514
1515 /*
1516 * copy the data from the old page to the new
1517 * one and clear the fake/clean flags on the
1518 * new page (keep it busy). force a reload
1519 * of the old page by clearing it from all
1520 * pmaps. then lock the page queues to
1521 * rename the pages.
1522 */
1523 uvm_pagecopy(uobjpage, pg); /* old -> new */
1524 atomic_clearbits_int(&pg->pg_flags,
1525 PG_FAKE|PG_CLEAN);
1526 pmap_page_protect(uobjpage, VM_PROT_NONE);
1527 if (uobjpage->pg_flags & PG_WANTED)
1528 wakeup(uobjpage);
1529 /* uobj still locked */
1530 atomic_clearbits_int(&uobjpage->pg_flags,
1531 PG_BUSY|PG_WANTED);
1532 UVM_PAGE_OWN(uobjpage, NULL);
1533
1534 uvm_lock_pageq();
1535 uoff = uobjpage->offset;
1536 /* remove old page */
1537 uvm_pagerealloc(uobjpage, NULL, 0);
1538
1539 /*
1540 * at this point we have absolutely no
1541 * control over uobjpage
1542 */
1543 /* install new page */
1544 uvm_pagerealloc(pg, uobj, uoff);
1545 uvm_unlock_pageq();
1546
1547 /*
1548 * done! loan is broken and "pg" is
1549 * PG_BUSY. it can now replace uobjpage.
1550 */
1551
1552 uobjpage = pg;
1553
1554 } /* write fault case */
1555 } /* if loan_count */
1556
1557 } else {
1558
1559 /*
1560 * if we are going to promote the data to an anon we
1561 * allocate a blank anon here and plug it into our amap.
1562 */
1563 #ifdef DIAGNOSTIC
1564 if (amap == NULL)
1565 panic("uvm_fault: want to promote data, but no anon");
1566 #endif
1567
1568 anon = uvm_analloc();
1569 if (anon) {
1570 /*
1571 * In `Fill in data...' below, if
1572 * uobjpage == PGO_DONTCARE, we want
1573 * a zero'd, dirty page, so have
1574 * uvm_pagealloc() do that for us.
1575 */
1576 pg = uvm_pagealloc(NULL, 0, anon,
1577 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0);
1578 }
1579
1580 /*
1581 * out of memory resources?
1582 */
1583 if (anon == NULL || pg == NULL) {
1584
1585 /*
1586 * arg! must unbusy our page and fail or sleep.
1587 */
1588 if (uobjpage != PGO_DONTCARE) {
1589 if (uobjpage->pg_flags & PG_WANTED)
1590 /* still holding object lock */
1591 wakeup(uobjpage);
1592
1593 uvm_lock_pageq();
1594 uvm_pageactivate(uobjpage);
1595 uvm_unlock_pageq();
1596 atomic_clearbits_int(&uobjpage->pg_flags,
1597 PG_BUSY|PG_WANTED);
1598 UVM_PAGE_OWN(uobjpage, NULL);
1599 }
1600
1601 /* unlock and fail ... */
1602 uvmfault_unlockall(&ufi, amap, uobj, NULL);
1603 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1604 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
1605 UVMHIST_LOG(maphist, " promote: out of VM",
1606 0,0,0,0);
1607 uvmexp.fltnoanon++;
1608 return (ENOMEM);
1609 }
1610
1611 UVMHIST_LOG(maphist, " out of RAM, waiting for more",
1612 0,0,0,0);
1613 uvm_anfree(anon);
1614 uvmexp.fltnoram++;
1615 uvm_wait("flt_noram5");
1616 goto ReFault;
1617 }
1618
1619 /*
1620 * fill in the data
1621 */
1622
1623 if (uobjpage != PGO_DONTCARE) {
1624 uvmexp.flt_prcopy++;
1625 /* copy page [pg now dirty] */
1626 uvm_pagecopy(uobjpage, pg);
1627
1628 /*
1629 * promote to shared amap? make sure all sharing
1630 * procs see it
1631 */
1632 if ((amap_flags(amap) & AMAP_SHARED) != 0) {
1633 pmap_page_protect(uobjpage, VM_PROT_NONE);
1634 }
1635
1636 /*
1637 * dispose of uobjpage. it can't be PG_RELEASED
1638 * since we still hold the object lock.
1639 * drop handle to uobj as well.
1640 */
1641
1642 if (uobjpage->pg_flags & PG_WANTED)
1643 /* still have the obj lock */
1644 wakeup(uobjpage);
1645 atomic_clearbits_int(&uobjpage->pg_flags,
1646 PG_BUSY|PG_WANTED);
1647 UVM_PAGE_OWN(uobjpage, NULL);
1648 uvm_lock_pageq();
1649 uvm_pageactivate(uobjpage);
1650 uvm_unlock_pageq();
1651 simple_unlock(&uobj->vmobjlock);
1652 uobj = NULL;
1653
1654 UVMHIST_LOG(maphist,
1655 " promote uobjpage %p to anon/page %p/%p",
1656 uobjpage, anon, pg, 0);
1657
1658 } else {
1659 uvmexp.flt_przero++;
1660 /*
1661 * Page is zero'd and marked dirty by uvm_pagealloc()
1662 * above.
1663 */
1664 UVMHIST_LOG(maphist," zero fill anon/page %p/%p",
1665 anon, pg, 0, 0);
1666 }
1667
1668 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
1669 anon, 0);
1670 }
1671
1672 /*
1673 * locked:
1674 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1675 *
1676 * note: pg is either the uobjpage or the new page in the new anon
1677 */
1678
1679 /*
1680 * all resources are present. we can now map it in and free our
1681 * resources.
1682 */
1683
1684 UVMHIST_LOG(maphist,
1685 " MAPPING: case2: pm=%p, va=0x%lx, pg=%p, promote=%ld",
1686 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);
1687 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1688 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
1689 != 0) {
1690
1691 /*
1692 * No need to undo what we did; we can simply think of
1693 * this as the pmap throwing away the mapping information.
1694 *
1695 * We do, however, have to go through the ReFault path,
1696 * as the map may change while we're asleep.
1697 */
1698
1699 if (pg->pg_flags & PG_WANTED)
1700 wakeup(pg); /* lock still held */
1701
1702 /*
1703 * note that pg can't be PG_RELEASED since we did not drop
1704 * the object lock since the last time we checked.
1705 */
1706
1707 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
1708 UVM_PAGE_OWN(pg, NULL);
1709 uvmfault_unlockall(&ufi, amap, uobj, NULL);
1710 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1711 if (uvmexp.swpgonly == uvmexp.swpages) {
1712 UVMHIST_LOG(maphist,
1713 "<- failed. out of VM",0,0,0,0);
1714 /* XXX instrumentation */
1715 return (ENOMEM);
1716 }
1717 /* XXX instrumentation */
1718 uvm_wait("flt_pmfail2");
1719 goto ReFault;
1720 }
1721
1722 uvm_lock_pageq();
1723
1724 if (fault_type == VM_FAULT_WIRE) {
1725 uvm_pagewire(pg);
1726 if (pg->pg_flags & PQ_AOBJ) {
1727
1728 /*
1729 * since the now-wired page cannot be paged out,
1730 * release its swap resources for others to use.
1731 * since an aobj page with no swap cannot be PG_CLEAN,
1732 * clear its clean flag now.
1733 */
1734 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
1735 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
1736 }
1737 } else {
1738 /* activate it */
1739 uvm_pageactivate(pg);
1740 }
1741 uvm_unlock_pageq();
1742
1743 if (pg->pg_flags & PG_WANTED)
1744 wakeup(pg); /* lock still held */
1745
1746 /*
1747 * note that pg can't be PG_RELEASED since we did not drop the object
1748 * lock since the last time we checked.
1749 */
1750
1751 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
1752 UVM_PAGE_OWN(pg, NULL);
1753 uvmfault_unlockall(&ufi, amap, uobj, NULL);
1754 pmap_update(ufi.orig_map->pmap);
1755
1756 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
1757 return (0);
1758 }
1759
1760
1761 /*
1762 * uvm_fault_wire: wire down a range of virtual addresses in a map.
1763 *
1764 * => map may be read-locked by caller, but MUST NOT be write-locked.
1765 * => if map is read-locked, any operations which may cause map to
1766 * be write-locked in uvm_fault() must be taken care of by
1767 * the caller. See uvm_map_pageable().
1768 */
1769
1770 int
1771 uvm_fault_wire(map, start, end, access_type)
1772 vm_map_t map;
1773 vaddr_t start, end;
1774 vm_prot_t access_type;
1775 {
1776 vaddr_t va;
1777 pmap_t pmap;
1778 int rv;
1779
1780 pmap = vm_map_pmap(map);
1781
1782 /*
1783 * now fault it in a page at a time. if the fault fails then we have
1784 * to undo what we have done. note that in uvm_fault VM_PROT_NONE
1785 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1786 */
1787
1788 for (va = start ; va < end ; va += PAGE_SIZE) {
1789 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type);
1790 if (rv) {
1791 if (va != start) {
1792 uvm_fault_unwire(map, start, va);
1793 }
1794 return (rv);
1795 }
1796 }
1797
1798 return (0);
1799 }
1800
1801 /*
1802 * uvm_fault_unwire(): unwire range of virtual space.
1803 */
1804
1805 void
1806 uvm_fault_unwire(map, start, end)
1807 vm_map_t map;
1808 vaddr_t start, end;
1809 {
1810
1811 vm_map_lock_read(map);
1812 uvm_fault_unwire_locked(map, start, end);
1813 vm_map_unlock_read(map);
1814 }
1815
1816 /*
1817 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
1818 *
1819 * => map must be at least read-locked.
1820 */
1821
1822 void
1823 uvm_fault_unwire_locked(map, start, end)
1824 vm_map_t map;
1825 vaddr_t start, end;
1826 {
1827 vm_map_entry_t entry;
1828 pmap_t pmap = vm_map_pmap(map);
1829 vaddr_t va;
1830 paddr_t pa;
1831 struct vm_page *pg;
1832
1833 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1834
1835 /*
1836 * we assume that the area we are unwiring has actually been wired
1837 * in the first place. this means that we should be able to extract
1838 * the PAs from the pmap. we also lock out the page daemon so that
1839 * we can call uvm_pageunwire.
1840 */
1841
1842 uvm_lock_pageq();
1843
1844 /*
1845 * find the beginning map entry for the region.
1846 */
1847 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1848 if (uvm_map_lookup_entry(map, start, &entry) == FALSE)
1849 panic("uvm_fault_unwire_locked: address not in map");
1850
1851 for (va = start; va < end ; va += PAGE_SIZE) {
1852 if (pmap_extract(pmap, va, &pa) == FALSE)
1853 continue;
1854
1855 /*
1856 * find the map entry for the current address.
1857 */
1858 KASSERT(va >= entry->start);
1859 while (va >= entry->end) {
1860 KASSERT(entry->next != &map->header &&
1861 entry->next->start <= entry->end);
1862 entry = entry->next;
1863 }
1864
1865 /*
1866 * if the entry is no longer wired, tell the pmap.
1867 */
1868 if (VM_MAPENT_ISWIRED(entry) == 0)
1869 pmap_unwire(pmap, va);
1870
1871 pg = PHYS_TO_VM_PAGE(pa);
1872 if (pg)
1873 uvm_pageunwire(pg);
1874 }
1875
1876 uvm_unlock_pageq();
1877 }