1 /* $OpenBSD: uvm_km.c,v 1.64 2007/08/03 22:49:07 art Exp $ */
2 /* $NetBSD: uvm_km.c,v 1.42 2001/01/14 02:10:01 thorpej Exp $ */
3
4 /*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * Copyright (c) 1991, 1993, The Regents of the University of California.
7 *
8 * All rights reserved.
9 *
10 * This code is derived from software contributed to Berkeley by
11 * The Mach Operating System project at Carnegie-Mellon University.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by Charles D. Cranor,
24 * Washington University, the University of California, Berkeley and
25 * its contributors.
26 * 4. Neither the name of the University nor the names of its contributors
27 * may be used to endorse or promote products derived from this software
28 * without specific prior written permission.
29 *
30 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 * SUCH DAMAGE.
41 *
42 * @(#)vm_kern.c 8.3 (Berkeley) 1/12/94
43 * from: Id: uvm_km.c,v 1.1.2.14 1998/02/06 05:19:27 chs Exp
44 *
45 *
46 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
47 * All rights reserved.
48 *
49 * Permission to use, copy, modify and distribute this software and
50 * its documentation is hereby granted, provided that both the copyright
51 * notice and this permission notice appear in all copies of the
52 * software, derivative works or modified versions, and any portions
53 * thereof, and that both notices appear in supporting documentation.
54 *
55 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58 *
59 * Carnegie Mellon requests users of this software to return to
60 *
61 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
62 * School of Computer Science
63 * Carnegie Mellon University
64 * Pittsburgh PA 15213-3890
65 *
66 * any improvements or extensions that they make and grant Carnegie the
67 * rights to redistribute these changes.
68 */
69
70 /*
71 * uvm_km.c: handle kernel memory allocation and management
72 */
73
74 /*
75 * overview of kernel memory management:
76 *
77 * the kernel virtual address space is mapped by "kernel_map." kernel_map
78 * starts at VM_MIN_KERNEL_ADDRESS and goes to VM_MAX_KERNEL_ADDRESS.
79 * note that VM_MIN_KERNEL_ADDRESS is equal to vm_map_min(kernel_map).
80 *
81 * the kernel_map has several "submaps." submaps can only appear in
82 * the kernel_map (user processes can't use them). submaps "take over"
83 * the management of a sub-range of the kernel's address space. submaps
84 * are typically allocated at boot time and are never released. kernel
85 * virtual address space that is mapped by a submap is locked by the
86 * submap's lock -- not the kernel_map's lock.
87 *
88 * thus, the useful feature of submaps is that they allow us to break
89 * up the locking and protection of the kernel address space into smaller
90 * chunks.
91 *
92 * the vm system has several standard kernel submaps, including:
93 * kmem_map => contains only wired kernel memory for the kernel
94 * malloc. *** access to kmem_map must be protected
95 * by splvm() because we are allowed to call malloc()
96 * at interrupt time ***
97 * pager_map => used to map "buf" structures into kernel space
98 * exec_map => used during exec to handle exec args
99 * etc...
100 *
101 * the kernel allocates its private memory out of special uvm_objects whose
102 * reference count is set to UVM_OBJ_KERN (thus indicating that the objects
103 * are "special" and never die). all kernel objects should be thought of
104 * as large, fixed-sized, sparsely populated uvm_objects. each kernel
105 * object is equal to the size of kernel virtual address space (i.e. the
106 * value "VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS").
107 *
108 * most kernel private memory lives in kernel_object. the only exception
109 * to this is for memory that belongs to submaps that must be protected
110 * by splvm(). each of these submaps manages their own pages.
111 *
112 * note that just because a kernel object spans the entire kernel virtual
113 * address space doesn't mean that it has to be mapped into the entire space.
114 * large chunks of a kernel object's space go unused either because
115 * that area of kernel VM is unmapped, or there is some other type of
116 * object mapped into that range (e.g. a vnode). for submap's kernel
117 * objects, the only part of the object that can ever be populated is the
118 * offsets that are managed by the submap.
119 *
120 * note that the "offset" in a kernel object is always the kernel virtual
121 * address minus the VM_MIN_KERNEL_ADDRESS (aka vm_map_min(kernel_map)).
122 * example:
123 * suppose VM_MIN_KERNEL_ADDRESS is 0xf8000000 and the kernel does a
124 * uvm_km_alloc(kernel_map, PAGE_SIZE) [allocate 1 wired down page in the
125 * kernel map]. if uvm_km_alloc returns virtual address 0xf8235000,
126 * then that means that the page at offset 0x235000 in kernel_object is
127 * mapped at 0xf8235000.
128 *
129 * kernel objects have one other special property: when the kernel virtual
130 * memory mapping them is unmapped, the backing memory in the object is
131 * freed right away. this is done with the uvm_km_pgremove() function.
132 * this has to be done because there is no backing store for kernel pages
133 * and no need to save them after they are no longer referenced.
134 */
135
136 #include <sys/param.h>
137 #include <sys/systm.h>
138 #include <sys/proc.h>
139 #include <sys/kthread.h>
140
141 #include <uvm/uvm.h>
142
143 /*
144 * global data structures
145 */
146
147 struct vm_map *kernel_map = NULL;
148
149 /*
150 * local data structues
151 */
152
153 static struct vm_map kernel_map_store;
154
155 /*
156 * uvm_km_init: init kernel maps and objects to reflect reality (i.e.
157 * KVM already allocated for text, data, bss, and static data structures).
158 *
159 * => KVM is defined by VM_MIN_KERNEL_ADDRESS/VM_MAX_KERNEL_ADDRESS.
160 * we assume that [min -> start] has already been allocated and that
161 * "end" is the end.
162 */
163
164 void
165 uvm_km_init(vaddr_t start, vaddr_t end)
166 {
167 vaddr_t base = VM_MIN_KERNEL_ADDRESS;
168
169 /*
170 * next, init kernel memory objects.
171 */
172
173 /* kernel_object: for pageable anonymous kernel memory */
174 uao_init();
175 uvm.kernel_object = uao_create(VM_MAX_KERNEL_ADDRESS -
176 VM_MIN_KERNEL_ADDRESS, UAO_FLAG_KERNOBJ);
177
178 /*
179 * init the map and reserve already allocated kernel space
180 * before installing.
181 */
182
183 uvm_map_setup(&kernel_map_store, base, end, VM_MAP_PAGEABLE);
184 kernel_map_store.pmap = pmap_kernel();
185 if (base != start && uvm_map(&kernel_map_store, &base, start - base,
186 NULL, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
187 UVM_INH_NONE, UVM_ADV_RANDOM,UVM_FLAG_FIXED)) != 0)
188 panic("uvm_km_init: could not reserve space for kernel");
189
190 /*
191 * install!
192 */
193
194 kernel_map = &kernel_map_store;
195 }
196
197 /*
198 * uvm_km_suballoc: allocate a submap in the kernel map. once a submap
199 * is allocated all references to that area of VM must go through it. this
200 * allows the locking of VAs in kernel_map to be broken up into regions.
201 *
202 * => if `fixed' is true, *min specifies where the region described
203 * by the submap must start
204 * => if submap is non NULL we use that as the submap, otherwise we
205 * alloc a new map
206 */
207 struct vm_map *
208 uvm_km_suballoc(struct vm_map *map, vaddr_t *min, vaddr_t *max, vsize_t size,
209 int flags, boolean_t fixed, struct vm_map *submap)
210 {
211 int mapflags = UVM_FLAG_NOMERGE | (fixed ? UVM_FLAG_FIXED : 0);
212
213 size = round_page(size); /* round up to pagesize */
214
215 /*
216 * first allocate a blank spot in the parent map
217 */
218
219 if (uvm_map(map, min, size, NULL, UVM_UNKNOWN_OFFSET, 0,
220 UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
221 UVM_ADV_RANDOM, mapflags)) != 0) {
222 panic("uvm_km_suballoc: unable to allocate space in parent map");
223 }
224
225 /*
226 * set VM bounds (min is filled in by uvm_map)
227 */
228
229 *max = *min + size;
230
231 /*
232 * add references to pmap and create or init the submap
233 */
234
235 pmap_reference(vm_map_pmap(map));
236 if (submap == NULL) {
237 submap = uvm_map_create(vm_map_pmap(map), *min, *max, flags);
238 if (submap == NULL)
239 panic("uvm_km_suballoc: unable to create submap");
240 } else {
241 uvm_map_setup(submap, *min, *max, flags);
242 submap->pmap = vm_map_pmap(map);
243 }
244
245 /*
246 * now let uvm_map_submap plug in it...
247 */
248
249 if (uvm_map_submap(map, *min, *max, submap) != 0)
250 panic("uvm_km_suballoc: submap allocation failed");
251
252 return(submap);
253 }
254
255 /*
256 * uvm_km_pgremove: remove pages from a kernel uvm_object.
257 *
258 * => when you unmap a part of anonymous kernel memory you want to toss
259 * the pages right away. (this gets called from uvm_unmap_...).
260 */
261 void
262 uvm_km_pgremove(struct uvm_object *uobj, vaddr_t start, vaddr_t end)
263 {
264 struct vm_page *pp;
265 voff_t curoff;
266 UVMHIST_FUNC("uvm_km_pgremove"); UVMHIST_CALLED(maphist);
267
268 KASSERT(uobj->pgops == &aobj_pager);
269
270 for (curoff = start ; curoff < end ; curoff += PAGE_SIZE) {
271 pp = uvm_pagelookup(uobj, curoff);
272 if (pp == NULL)
273 continue;
274
275 UVMHIST_LOG(maphist," page %p, busy=%ld", pp,
276 pp->pg_flags & PG_BUSY, 0, 0);
277
278 if (pp->pg_flags & PG_BUSY) {
279 /* owner must check for this when done */
280 atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
281 } else {
282 /* free the swap slot... */
283 uao_dropswap(uobj, curoff >> PAGE_SHIFT);
284
285 /*
286 * ...and free the page; note it may be on the
287 * active or inactive queues.
288 */
289 uvm_lock_pageq();
290 uvm_pagefree(pp);
291 uvm_unlock_pageq();
292 }
293 }
294 }
295
296
297 /*
298 * uvm_km_pgremove_intrsafe: like uvm_km_pgremove(), but for "intrsafe"
299 * objects
300 *
301 * => when you unmap a part of anonymous kernel memory you want to toss
302 * the pages right away. (this gets called from uvm_unmap_...).
303 * => none of the pages will ever be busy, and none of them will ever
304 * be on the active or inactive queues (because these objects are
305 * never allowed to "page").
306 */
307
308 void
309 uvm_km_pgremove_intrsafe(vaddr_t start, vaddr_t end)
310 {
311 struct vm_page *pg;
312 vaddr_t va;
313 paddr_t pa;
314
315 for (va = start; va < end; va += PAGE_SIZE) {
316 if (!pmap_extract(pmap_kernel(), va, &pa))
317 continue;
318 pg = PHYS_TO_VM_PAGE(pa);
319 if (pg == NULL)
320 panic("uvm_km_pgremove_intrsafe: no page");
321 uvm_pagefree(pg);
322 }
323 }
324
325
326 /*
327 * uvm_km_kmemalloc: lower level kernel memory allocator for malloc()
328 *
329 * => we map wired memory into the specified map using the obj passed in
330 * => NOTE: we can return NULL even if we can wait if there is not enough
331 * free VM space in the map... caller should be prepared to handle
332 * this case.
333 * => we return KVA of memory allocated
334 * => flags: NOWAIT, VALLOC - just allocate VA, TRYLOCK - fail if we can't
335 * lock the map
336 */
337
338 vaddr_t
339 uvm_km_kmemalloc(struct vm_map *map, struct uvm_object *obj, vsize_t size,
340 int flags)
341 {
342 vaddr_t kva, loopva;
343 voff_t offset;
344 struct vm_page *pg;
345 UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist);
346
347 UVMHIST_LOG(maphist," (map=%p, obj=%p, size=0x%lx, flags=%d)",
348 map, obj, size, flags);
349 KASSERT(vm_map_pmap(map) == pmap_kernel());
350
351 /*
352 * setup for call
353 */
354
355 size = round_page(size);
356 kva = vm_map_min(map); /* hint */
357
358 /*
359 * allocate some virtual space
360 */
361
362 if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
363 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
364 UVM_ADV_RANDOM, (flags & UVM_KMF_TRYLOCK))) != 0)) {
365 UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
366 return(0);
367 }
368
369 /*
370 * if all we wanted was VA, return now
371 */
372
373 if (flags & UVM_KMF_VALLOC) {
374 UVMHIST_LOG(maphist,"<- done valloc (kva=0x%lx)", kva,0,0,0);
375 return(kva);
376 }
377
378 /*
379 * recover object offset from virtual address
380 */
381
382 if (obj != NULL)
383 offset = kva - vm_map_min(kernel_map);
384 else
385 offset = 0;
386
387 UVMHIST_LOG(maphist, " kva=0x%lx, offset=0x%lx", kva, offset,0,0);
388
389 /*
390 * now allocate and map in the memory... note that we are the only ones
391 * whom should ever get a handle on this area of VM.
392 */
393
394 loopva = kva;
395 while (loopva != kva + size) {
396 pg = uvm_pagealloc(obj, offset, NULL, 0);
397 if (pg) {
398 atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
399 UVM_PAGE_OWN(pg, NULL);
400 }
401
402 if (__predict_false(pg == NULL)) {
403 if ((flags & UVM_KMF_NOWAIT) ||
404 ((flags & UVM_KMF_CANFAIL) &&
405 uvmexp.swpgonly == uvmexp.swpages)) {
406 /* free everything! */
407 uvm_unmap(map, kva, kva + size);
408 return (0);
409 } else {
410 uvm_wait("km_getwait2"); /* sleep here */
411 continue;
412 }
413 }
414
415 /*
416 * map it in: note that we call pmap_enter with the map and
417 * object unlocked in case we are kmem_map.
418 */
419
420 if (obj == NULL) {
421 pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
422 UVM_PROT_RW);
423 } else {
424 pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
425 UVM_PROT_RW,
426 PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
427 }
428 loopva += PAGE_SIZE;
429 offset += PAGE_SIZE;
430 }
431 pmap_update(pmap_kernel());
432
433 UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
434 return(kva);
435 }
436
437 /*
438 * uvm_km_free: free an area of kernel memory
439 */
440
441 void
442 uvm_km_free(struct vm_map *map, vaddr_t addr, vsize_t size)
443 {
444 uvm_unmap(map, trunc_page(addr), round_page(addr+size));
445 }
446
447 /*
448 * uvm_km_free_wakeup: free an area of kernel memory and wake up
449 * anyone waiting for vm space.
450 *
451 * => XXX: "wanted" bit + unlock&wait on other end?
452 */
453
454 void
455 uvm_km_free_wakeup(struct vm_map *map, vaddr_t addr, vsize_t size)
456 {
457 struct vm_map_entry *dead_entries;
458
459 vm_map_lock(map);
460 uvm_unmap_remove(map, trunc_page(addr), round_page(addr+size),
461 &dead_entries, NULL);
462 wakeup(map);
463 vm_map_unlock(map);
464
465 if (dead_entries != NULL)
466 uvm_unmap_detach(dead_entries, 0);
467 }
468
469 /*
470 * uvm_km_alloc1: allocate wired down memory in the kernel map.
471 *
472 * => we can sleep if needed
473 */
474
475 vaddr_t
476 uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit)
477 {
478 vaddr_t kva, loopva;
479 voff_t offset;
480 struct vm_page *pg;
481 UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist);
482
483 UVMHIST_LOG(maphist,"(map=%p, size=0x%lx)", map, size,0,0);
484 KASSERT(vm_map_pmap(map) == pmap_kernel());
485
486 size = round_page(size);
487 kva = vm_map_min(map); /* hint */
488
489 /*
490 * allocate some virtual space
491 */
492
493 if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
494 UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
495 UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) {
496 UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0);
497 return(0);
498 }
499
500 /*
501 * recover object offset from virtual address
502 */
503
504 offset = kva - vm_map_min(kernel_map);
505 UVMHIST_LOG(maphist," kva=0x%lx, offset=0x%lx", kva, offset,0,0);
506
507 /*
508 * now allocate the memory. we must be careful about released pages.
509 */
510
511 loopva = kva;
512 while (size) {
513 simple_lock(&uvm.kernel_object->vmobjlock);
514 pg = uvm_pagelookup(uvm.kernel_object, offset);
515
516 /*
517 * if we found a page in an unallocated region, it must be
518 * released
519 */
520 if (pg) {
521 if ((pg->pg_flags & PG_RELEASED) == 0)
522 panic("uvm_km_alloc1: non-released page");
523 atomic_setbits_int(&pg->pg_flags, PG_WANTED);
524 UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
525 FALSE, "km_alloc", 0);
526 continue; /* retry */
527 }
528
529 /* allocate ram */
530 pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
531 if (pg) {
532 atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
533 UVM_PAGE_OWN(pg, NULL);
534 }
535 simple_unlock(&uvm.kernel_object->vmobjlock);
536 if (__predict_false(pg == NULL)) {
537 if (curproc == uvm.pagedaemon_proc) {
538 /*
539 * It is unfeasible for the page daemon to
540 * sleep for memory, so free what we have
541 * allocated and fail.
542 */
543 uvm_unmap(map, kva, loopva - kva);
544 return (NULL);
545 } else {
546 uvm_wait("km_alloc1w"); /* wait for memory */
547 continue;
548 }
549 }
550
551 /*
552 * map it in; note we're never called with an intrsafe
553 * object, so we always use regular old pmap_enter().
554 */
555 pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
556 UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
557
558 loopva += PAGE_SIZE;
559 offset += PAGE_SIZE;
560 size -= PAGE_SIZE;
561 }
562 pmap_update(map->pmap);
563
564 /*
565 * zero on request (note that "size" is now zero due to the above loop
566 * so we need to subtract kva from loopva to reconstruct the size).
567 */
568
569 if (zeroit)
570 memset((caddr_t)kva, 0, loopva - kva);
571
572 UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
573 return(kva);
574 }
575
576 /*
577 * uvm_km_valloc: allocate zero-fill memory in the kernel's address space
578 *
579 * => memory is not allocated until fault time
580 */
581
582 vaddr_t
583 uvm_km_valloc(struct vm_map *map, vsize_t size)
584 {
585 return(uvm_km_valloc_align(map, size, 0));
586 }
587
588 vaddr_t
589 uvm_km_valloc_align(struct vm_map *map, vsize_t size, vsize_t align)
590 {
591 vaddr_t kva;
592 UVMHIST_FUNC("uvm_km_valloc"); UVMHIST_CALLED(maphist);
593
594 UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0);
595 KASSERT(vm_map_pmap(map) == pmap_kernel());
596
597 size = round_page(size);
598 kva = vm_map_min(map); /* hint */
599
600 /*
601 * allocate some virtual space. will be demand filled by kernel_object.
602 */
603
604 if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
605 UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
606 UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) {
607 UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
608 return(0);
609 }
610
611 UVMHIST_LOG(maphist, "<- done (kva=0x%lx)", kva,0,0,0);
612 return(kva);
613 }
614
615 /*
616 * uvm_km_valloc_wait: allocate zero-fill memory in the kernel's address space
617 *
618 * => memory is not allocated until fault time
619 * => if no room in map, wait for space to free, unless requested size
620 * is larger than map (in which case we return 0)
621 */
622
623 vaddr_t
624 uvm_km_valloc_prefer_wait(struct vm_map *map, vsize_t size, voff_t prefer)
625 {
626 vaddr_t kva;
627 UVMHIST_FUNC("uvm_km_valloc_prefer_wait"); UVMHIST_CALLED(maphist);
628
629 UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0);
630 KASSERT(vm_map_pmap(map) == pmap_kernel());
631
632 size = round_page(size);
633 if (size > vm_map_max(map) - vm_map_min(map))
634 return(0);
635
636 while (1) {
637 kva = vm_map_min(map); /* hint */
638
639 /*
640 * allocate some virtual space. will be demand filled
641 * by kernel_object.
642 */
643
644 if (__predict_true(uvm_map(map, &kva, size, uvm.kernel_object,
645 prefer, 0, UVM_MAPFLAG(UVM_PROT_ALL,
646 UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0)) == 0)) {
647 UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
648 return(kva);
649 }
650
651 /*
652 * failed. sleep for a while (on map)
653 */
654
655 UVMHIST_LOG(maphist,"<<<sleeping>>>",0,0,0,0);
656 tsleep((caddr_t)map, PVM, "vallocwait", 0);
657 }
658 /*NOTREACHED*/
659 }
660
661 vaddr_t
662 uvm_km_valloc_wait(struct vm_map *map, vsize_t size)
663 {
664 return uvm_km_valloc_prefer_wait(map, size, UVM_UNKNOWN_OFFSET);
665 }
666
667 /*
668 * uvm_km_alloc_poolpage: allocate a page for the pool allocator
669 *
670 * => if the pmap specifies an alternate mapping method, we use it.
671 */
672
673 /* ARGSUSED */
674 vaddr_t
675 uvm_km_alloc_poolpage1(struct vm_map *map, struct uvm_object *obj,
676 boolean_t waitok)
677 {
678 #if defined(__HAVE_PMAP_DIRECT)
679 struct vm_page *pg;
680 vaddr_t va;
681
682 again:
683 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
684 if (__predict_false(pg == NULL)) {
685 if (waitok) {
686 uvm_wait("plpg");
687 goto again;
688 } else
689 return (0);
690 }
691 va = pmap_map_direct(pg);
692 if (__predict_false(va == 0))
693 uvm_pagefree(pg);
694 return (va);
695 #else
696 vaddr_t va;
697 int s;
698
699 /*
700 * NOTE: We may be called with a map that doesn't require splvm
701 * protection (e.g. kernel_map). However, it does not hurt to
702 * go to splvm in this case (since unprotected maps will never be
703 * accessed in interrupt context).
704 *
705 * XXX We may want to consider changing the interface to this
706 * XXX function.
707 */
708
709 s = splvm();
710 va = uvm_km_kmemalloc(map, obj, PAGE_SIZE, waitok ? 0 : UVM_KMF_NOWAIT);
711 splx(s);
712 return (va);
713 #endif /* __HAVE_PMAP_DIRECT */
714 }
715
716 /*
717 * uvm_km_free_poolpage: free a previously allocated pool page
718 *
719 * => if the pmap specifies an alternate unmapping method, we use it.
720 */
721
722 /* ARGSUSED */
723 void
724 uvm_km_free_poolpage1(struct vm_map *map, vaddr_t addr)
725 {
726 #if defined(__HAVE_PMAP_DIRECT)
727 uvm_pagefree(pmap_unmap_direct(addr));
728 #else
729 int s;
730
731 /*
732 * NOTE: We may be called with a map that doesn't require splvm
733 * protection (e.g. kernel_map). However, it does not hurt to
734 * go to splvm in this case (since unprocted maps will never be
735 * accessed in interrupt context).
736 *
737 * XXX We may want to consider changing the interface to this
738 * XXX function.
739 */
740
741 s = splvm();
742 uvm_km_free(map, addr, PAGE_SIZE);
743 splx(s);
744 #endif /* __HAVE_PMAP_DIRECT */
745 }
746
747 #if defined(__HAVE_PMAP_DIRECT)
748 /*
749 * uvm_km_page allocator, __HAVE_PMAP_DIRECT arch
750 * On architectures with machine memory direct mapped into a portion
751 * of KVM, we have very little work to do. Just get a physical page,
752 * and find and return its VA. We use the poolpage functions for this.
753 */
754 void
755 uvm_km_page_init(void)
756 {
757 /* nothing */
758 }
759
760 void *
761 uvm_km_getpage(boolean_t waitok)
762 {
763
764 return ((void *)uvm_km_alloc_poolpage1(NULL, NULL, waitok));
765 }
766
767 void
768 uvm_km_putpage(void *v)
769 {
770
771 uvm_km_free_poolpage1(NULL, (vaddr_t)v);
772 }
773
774 #else
775 /*
776 * uvm_km_page allocator, non __HAVE_PMAP_DIRECT archs
777 * This is a special allocator that uses a reserve of free pages
778 * to fulfill requests. It is fast and interrupt safe, but can only
779 * return page sized regions. Its primary use is as a backend for pool.
780 *
781 * The memory returned is allocated from the larger kernel_map, sparing
782 * pressure on the small interrupt-safe kmem_map. It is wired, but
783 * not zero filled.
784 */
785
786 int uvm_km_pages_lowat; /* allocate more when reserve drops below this */
787 int uvm_km_pages_free; /* number of pages currently on free list */
788 struct km_page {
789 struct km_page *next;
790 } *uvm_km_pages_head;
791
792 void uvm_km_createthread(void *);
793 void uvm_km_thread(void *);
794
795 /*
796 * Allocate the initial reserve, and create the thread which will
797 * keep the reserve full. For bootstrapping, we allocate more than
798 * the lowat amount, because it may be a while before the thread is
799 * running.
800 */
801 void
802 uvm_km_page_init(void)
803 {
804 struct km_page *page;
805 int i;
806
807 if (!uvm_km_pages_lowat) {
808 /* based on physmem, calculate a good value here */
809 uvm_km_pages_lowat = physmem / 256;
810 if (uvm_km_pages_lowat > 2048)
811 uvm_km_pages_lowat = 2048;
812 if (uvm_km_pages_lowat < 128)
813 uvm_km_pages_lowat = 128;
814 }
815
816 for (i = 0; i < uvm_km_pages_lowat * 4; i++) {
817 page = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE);
818 page->next = uvm_km_pages_head;
819 uvm_km_pages_head = page;
820 }
821 uvm_km_pages_free = i;
822
823 /* tone down if really high */
824 if (uvm_km_pages_lowat > 512)
825 uvm_km_pages_lowat = 512;
826
827 kthread_create_deferred(uvm_km_createthread, NULL);
828 }
829
830 void
831 uvm_km_createthread(void *arg)
832 {
833 kthread_create(uvm_km_thread, NULL, NULL, "kmthread");
834 }
835
836 /*
837 * Endless loop. We grab pages in increments of 16 pages, then
838 * quickly swap them into the list. At some point we can consider
839 * returning memory to the system if we have too many free pages,
840 * but that's not implemented yet.
841 */
842 void
843 uvm_km_thread(void *arg)
844 {
845 struct km_page *head, *tail, *page;
846 int i, s, want;
847
848 for (i = want = 16; ; ) {
849 if (i < want || uvm_km_pages_free >= uvm_km_pages_lowat)
850 tsleep(&uvm_km_pages_head, PVM, "kmalloc", 0);
851 for (i = 0; i < want; i++) {
852 page = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE);
853 if (i == 0)
854 head = tail = page;
855 if (page == NULL)
856 break;
857 page->next = head;
858 head = page;
859 }
860 if (head != NULL) {
861 s = splvm();
862 tail->next = uvm_km_pages_head;
863 uvm_km_pages_head = head;
864 uvm_km_pages_free += i;
865 splx(s);
866 }
867 if (uvm_km_pages_free)
868 wakeup(&uvm_km_pages_free);
869 }
870 }
871
872
873 /*
874 * Allocate one page. We can sleep for more if the caller
875 * permits it. Wake up the thread if we've dropped below lowat.
876 */
877 void *
878 uvm_km_getpage(boolean_t waitok)
879 {
880 struct km_page *page = NULL;
881 int s;
882
883 s = splvm();
884 for (;;) {
885 page = uvm_km_pages_head;
886 if (page) {
887 uvm_km_pages_head = page->next;
888 uvm_km_pages_free--;
889 break;
890 }
891 if (!waitok)
892 break;
893 tsleep(&uvm_km_pages_free, PVM, "getpage", 0);
894 }
895 splx(s);
896 if (uvm_km_pages_free < uvm_km_pages_lowat)
897 wakeup(&uvm_km_pages_head);
898 return (page);
899 }
900
901 void
902 uvm_km_putpage(void *v)
903 {
904 struct km_page *page = v;
905 int s;
906
907 s = splvm();
908 page->next = uvm_km_pages_head;
909 uvm_km_pages_head = page;
910 uvm_km_pages_free++;
911 splx(s);
912 }
913 #endif