root/kern/subr_pool.c

DEFINITIONS

1. pr_log
2. pr_printlog
3. pr_enter
4. pr_leave
5. pr_enter_check
6. phtree_compare
7. pr_find_pagehead
8. pr_rmpage
9. pool_init
10. pool_setipl
11. pool_destroy
12. pool_alloc_item_header
13. _pool_get
14. pool_do_put
15. _pool_put
16. pool_put
17. pool_prime
18. pool_prime_page
19. pool_catchup
20. pool_update_curpage
21. pool_setlowat
22. pool_sethiwat
23. pool_sethardlimit
24. _pool_reclaim
25. pool_printit
26. pool_print_pagelist
27. pool_print1
28. db_show_all_pools
29. pool_chk_page
30. pool_chk
31. pool_cache_init
32. pool_cache_destroy
33. pcg_get
34. pcg_put
35. pool_cache_get
36. pool_cache_put
37. pool_cache_destruct_object
38. pool_cache_do_invalidate
39. pool_cache_invalidate
40. pool_cache_reclaim
41. sysctl_dopool
42. pool_allocator_alloc
43. pool_allocator_free
44. pool_page_alloc
45. pool_page_free
46. pool_page_alloc_oldnointr
47. pool_page_free_oldnointr

    1 /*      $OpenBSD: subr_pool.c,v 1.55 2007/08/16 15:18:54 art Exp $      */
    2 /*      $NetBSD: subr_pool.c,v 1.61 2001/09/26 07:14:56 chs Exp $       */
    4 /*-
    5  * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc.
    6  * All rights reserved.
    7  *
    8  * This code is derived from software contributed to The NetBSD Foundation
    9  * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
   10  * Simulation Facility, NASA Ames Research Center.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the NetBSD
   23  *      Foundation, Inc. and its contributors.
   24  * 4. Neither the name of The NetBSD Foundation nor the names of its
   25  *    contributors may be used to endorse or promote products derived
   26  *    from this software without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
   29  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   30  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   31  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
   32  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   33  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   34  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   35  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   36  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   37  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   38  * POSSIBILITY OF SUCH DAMAGE.
   39  */
   41 #include <sys/param.h>
   42 #include <sys/systm.h>
   43 #include <sys/proc.h>
   44 #include <sys/errno.h>
   45 #include <sys/kernel.h>
   46 #include <sys/malloc.h>
   47 #include <sys/lock.h>
   48 #include <sys/pool.h>
   49 #include <sys/syslog.h>
   50 #include <sys/sysctl.h>
   52 #include <uvm/uvm.h>
   54 /*
   55  * XXX - for now.
   56  */
   57 #ifdef LOCKDEBUG
   58 #define simple_lock_freecheck(a, s) do { /* nothing */ } while (0)
   59 #define simple_lock_only_held(lkp, str) do { /* nothing */ } while (0)
   60 #endif
   62 /*
   63  * Pool resource management utility.
   64  *
   65  * Memory is allocated in pages which are split into pieces according to
   66  * the pool item size. Each page is kept on one of three lists in the
   67  * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages',
   68  * for empty, full and partially-full pages respectively. The individual
   69  * pool items are on a linked list headed by `ph_itemlist' in each page
   70  * header. The memory for building the page list is either taken from
   71  * the allocated pages themselves (for small pool items) or taken from
   72  * an internal pool of page headers (`phpool').
   73  */
   75 /* List of all pools */
TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
   78 /* Private pool for page header structures */
   79 static struct pool phpool;
   81 /* This spin lock protects both pool_head */
   82 struct simplelock pool_head_slock;
   84 struct pool_item_header {
   85         /* Page headers */
LIST_ENTRY(pool_item_header)
ph_pagelist;    /* pool page list */
TAILQ_HEAD(,pool_item)  ph_itemlist;    /* chunk list for this page */
SPLAY_ENTRY(pool_item_header)
ph_node;        /* Off-page page headers */
   91         int                     ph_nmissing;    /* # of chunks in use */
caddr_t                 ph_page;        /* this page's address */
   93 };
   95 struct pool_item {
   96 #ifdef DIAGNOSTIC
   97         int pi_magic;
   98 #endif
   99 #ifdef DEADBEEF1
  100 #define PI_MAGIC DEADBEEF1
  101 #else
  102 #define PI_MAGIC 0xdeafbeef
  103 #endif
  104         /* Other entries use only this list entry */
TAILQ_ENTRY(pool_item)  pi_list;
  106 };
  108 #define POOL_NEEDS_CATCHUP(pp)                                          \
  109         ((pp)->pr_nitems < (pp)->pr_minitems)
  111 /*
  112  * Every pool gets a unique serial number assigned to it. If this counter
  113  * wraps, we're screwed, but we shouldn't create so many pools anyway.
  114  */
  115 unsigned int pool_serial;
  117 /*
  118  * Pool cache management.
  119  *
  120  * Pool caches provide a way for constructed objects to be cached by the
  121  * pool subsystem.  This can lead to performance improvements by avoiding
  122  * needless object construction/destruction; it is deferred until absolutely
  123  * necessary.
  124  *
  125  * Caches are grouped into cache groups.  Each cache group references
  126  * up to 16 constructed objects.  When a cache allocates an object
  127  * from the pool, it calls the object's constructor and places it into
  128  * a cache group.  When a cache group frees an object back to the pool,
  129  * it first calls the object's destructor.  This allows the object to
  130  * persist in constructed form while freed to the cache.
  131  *
  132  * Multiple caches may exist for each pool.  This allows a single
  133  * object type to have multiple constructed forms.  The pool references
  134  * each cache, so that when a pool is drained by the pagedaemon, it can
  135  * drain each individual cache as well.  Each time a cache is drained,
  136  * the most idle cache group is freed to the pool in its entirety.
  137  *
  138  * Pool caches are layed on top of pools.  By layering them, we can avoid
  139  * the complexity of cache management for pools which would not benefit
  140  * from it.
  141  */
  143 /* The cache group pool. */
  144 static struct pool pcgpool;
  146 /* The pool cache group. */
  147 #define PCG_NOBJECTS            16
  148 struct pool_cache_group {
TAILQ_ENTRY(pool_cache_group)
pcg_list;       /* link in the pool cache's group list */
u_int   pcg_avail;      /* # available objects */
  152                                 /* pointers to the objects */
  153         void    *pcg_objects[PCG_NOBJECTS];
  154 };
  156 void    pool_cache_reclaim(struct pool_cache *);
  157 void    pool_cache_do_invalidate(struct pool_cache *, int,
  158     void (*)(struct pool *, void *));
  160 int     pool_catchup(struct pool *);
  161 void    pool_prime_page(struct pool *, caddr_t, struct pool_item_header *);
  162 void    pool_update_curpage(struct pool *);
  163 void    pool_do_put(struct pool *, void *);
  164 void    pr_rmpage(struct pool *, struct pool_item_header *,
  165     struct pool_pagelist *);
  166 int     pool_chk_page(struct pool *, const char *, struct pool_item_header *);
  168 void    *pool_allocator_alloc(struct pool *, int);
  169 void    pool_allocator_free(struct pool *, void *);
  171 #ifdef DDB
  172 void pool_print_pagelist(struct pool_pagelist *, int (*)(const char *, ...));
  173 void pool_print1(struct pool *, const char *, int (*)(const char *, ...));
  174 #endif
  177 /*
  178  * Pool log entry. An array of these is allocated in pool_init().
  179  */
  180 struct pool_log {
  181         const char      *pl_file;
  182         long            pl_line;
  183         int             pl_action;
  184 #define PRLOG_GET       1
  185 #define PRLOG_PUT       2
  186         void            *pl_addr;
  187 };
  189 /* Number of entries in pool log buffers */
  190 #ifndef POOL_LOGSIZE
  191 #define POOL_LOGSIZE    10
  192 #endif
  194 int pool_logsize = POOL_LOGSIZE;
  196 #ifdef POOL_DIAGNOSTIC
  197 static __inline void
pr_log(struct pool *pp, void *v, int action, const char *file, long line)
  199 {
  200         int n = pp->pr_curlogentry;
  201         struct pool_log *pl;
  203         if ((pp->pr_roflags & PR_LOGGING) == 0)
  204                 return;
  206         /*
  207          * Fill in the current entry. Wrap around and overwrite
  208          * the oldest entry if necessary.
  209          */
pl = &pp->pr_log[n];
pl->pl_file = file;
pl->pl_line = line;
pl->pl_action = action;
pl->pl_addr = v;
  215         if (++n >= pp->pr_logsize)
n = 0;
pp->pr_curlogentry = n;
  218 }
  220 static void
pr_printlog(struct pool *pp, struct pool_item *pi,
  222     int (*pr)(const char *, ...))
  223 {
  224         int i = pp->pr_logsize;
  225         int n = pp->pr_curlogentry;
  227         if ((pp->pr_roflags & PR_LOGGING) == 0)
  228                 return;
  230         /*
  231          * Print all entries in this pool's log.
  232          */
  233         while (i-- > 0) {
  234                 struct pool_log *pl = &pp->pr_log[n];
  235                 if (pl->pl_action != 0) {
  236                         if (pi == NULL || pi == pl->pl_addr) {
  237                                 (*pr)("\tlog entry %d:\n", i);
  238                                 (*pr)("\t\taction = %s, addr = %p\n",
pl->pl_action == PRLOG_GET ? "get" : "put",
pl->pl_addr);
  241                                 (*pr)("\t\tfile: %s at line %lu\n",
pl->pl_file, pl->pl_line);
  243                         }
  244                 }
  245                 if (++n >= pp->pr_logsize)
n = 0;
  247         }
  248 }
  250 static __inline void
pr_enter(struct pool *pp, const char *file, long line)
  252 {
  254         if (__predict_false(pp->pr_entered_file != NULL)) {
printf("pool %s: reentrancy at file %s line %ld\n",
pp->pr_wchan, file, line);
printf("         previous entry at file %s line %ld\n",
pp->pr_entered_file, pp->pr_entered_line);
panic("pr_enter");
  260         }
pp->pr_entered_file = file;
pp->pr_entered_line = line;
  264 }
  266 static __inline void
pr_leave(struct pool *pp)
  268 {
  270         if (__predict_false(pp->pr_entered_file == NULL)) {
printf("pool %s not entered?\n", pp->pr_wchan);
panic("pr_leave");
  273         }
pp->pr_entered_file = NULL;
pp->pr_entered_line = 0;
  277 }
  279 static __inline void
pr_enter_check(struct pool *pp, int (*pr)(const char *, ...))
  281 {
  283         if (pp->pr_entered_file != NULL)
  284                 (*pr)("\n\tcurrently entered from file %s line %ld\n",
pp->pr_entered_file, pp->pr_entered_line);
  286 }
  287 #else
  288 #define pr_log(pp, v, action, file, line)
  289 #define pr_printlog(pp, pi, pr)
  290 #define pr_enter(pp, file, line)
  291 #define pr_leave(pp)
  292 #define pr_enter_check(pp, pr)
  293 #endif /* POOL_DIAGNOSTIC */
  295 static __inline int
  296 phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
  297 {
  298         if (a->ph_page < b->ph_page)
  299                 return (-1);
  300         else if (a->ph_page > b->ph_page)
  301                 return (1);
  302         else
  303                 return (0);
  304 }
SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare);
SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare);
  309 /*
  310  * Return the pool page header based on page address.
  311  */
  312 static __inline struct pool_item_header *
pr_find_pagehead(struct pool *pp, caddr_t page)
  314 {
  315         struct pool_item_header *ph, tmp;
  317         if ((pp->pr_roflags & PR_PHINPAGE) != 0)
  318                 return ((struct pool_item_header *)(page + pp->pr_phoffset));
tmp.ph_page = page;
ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
  322         return ph;
  323 }
  325 /*
  326  * Remove a page from the pool.
  327  */
  328 void
pr_rmpage(struct pool *pp, struct pool_item_header *ph,
  330      struct pool_pagelist *pq)
  331 {
  332         int s;
  334         /*
  335          * If the page was idle, decrement the idle page count.
  336          */
  337         if (ph->ph_nmissing == 0) {
  338 #ifdef DIAGNOSTIC
  339                 if (pp->pr_nidle == 0)
panic("pr_rmpage: nidle inconsistent");
  341                 if (pp->pr_nitems < pp->pr_itemsperpage)
panic("pr_rmpage: nitems inconsistent");
  343 #endif
pp->pr_nidle--;
  345         }
pp->pr_nitems -= pp->pr_itemsperpage;
  349         /*
  350          * Unlink a page from the pool and release it (or queue it for release).
  351          */
LIST_REMOVE(ph, ph_pagelist);
  353         if (pq) {
LIST_INSERT_HEAD(pq, ph, ph_pagelist);
  355         } else {
pool_allocator_free(pp, ph->ph_page);
  357                 if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
s = splhigh();
pool_put(&phpool, ph);
splx(s);
  362                 }
  363         }
pp->pr_npages--;
pp->pr_npagefree++;
pool_update_curpage(pp);
  368 }
  370 /*
  371  * Initialize the given pool resource structure.
  372  *
  373  * We export this routine to allow other kernel parts to declare
  374  * static pools that must be initialized before malloc() is available.
  375  */
  376 void
pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
  378     const char *wchan, struct pool_allocator *palloc)
  379 {
  380         int off, slack;
  382 #ifdef POOL_DIAGNOSTIC
  383         /*
  384          * Always log if POOL_DIAGNOSTIC is defined.
  385          */
  386         if (pool_logsize != 0)
flags |= PR_LOGGING;
  388 #endif
  390 #ifdef MALLOC_DEBUG
  391         if ((flags & PR_DEBUG) && (ioff != 0 || align != 0))
flags &= ~PR_DEBUG;
  393 #endif
  394         /*
  395          * Check arguments and construct default values.
  396          */
  397         if (palloc == NULL)
palloc = &pool_allocator_nointr;
  399         if ((palloc->pa_flags & PA_INITIALIZED) == 0) {
  400                 if (palloc->pa_pagesz == 0)
palloc->pa_pagesz = PAGE_SIZE;
TAILQ_INIT(&palloc->pa_list);
simple_lock_init(&palloc->pa_slock);
palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);
palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;
palloc->pa_flags |= PA_INITIALIZED;
  409         }
  411         if (align == 0)
align = ALIGN(1);
  414         if (size < sizeof(struct pool_item))
size = sizeof(struct pool_item);
size = roundup(size, align);
  418 #ifdef DIAGNOSTIC
  419         if (size > palloc->pa_pagesz)
panic("pool_init: pool item size (%lu) too large",
  421                       (u_long)size);
  422 #endif
  424         /*
  425          * Initialize the pool structure.
  426          */
LIST_INIT(&pp->pr_emptypages);
LIST_INIT(&pp->pr_fullpages);
LIST_INIT(&pp->pr_partpages);
TAILQ_INIT(&pp->pr_cachelist);
pp->pr_curpage = NULL;
pp->pr_npages = 0;
pp->pr_minitems = 0;
pp->pr_minpages = 0;
pp->pr_maxpages = 8;
pp->pr_roflags = flags;
pp->pr_flags = 0;
pp->pr_size = size;
pp->pr_align = align;
pp->pr_wchan = wchan;
pp->pr_alloc = palloc;
pp->pr_nitems = 0;
pp->pr_nout = 0;
pp->pr_hardlimit = UINT_MAX;
pp->pr_hardlimit_warning = NULL;
pp->pr_hardlimit_ratecap.tv_sec = 0;
pp->pr_hardlimit_ratecap.tv_usec = 0;
pp->pr_hardlimit_warning_last.tv_sec = 0;
pp->pr_hardlimit_warning_last.tv_usec = 0;
pp->pr_serial = ++pool_serial;
  451         if (pool_serial == 0)
panic("pool_init: too much uptime");
  454         /*
  455          * Decide whether to put the page header off page to avoid
  456          * wasting too large a part of the page. Off-page page headers
  457          * go on a hash table, so we can match a returned item
  458          * with its header based on the page address.
  459          * We use 1/16 of the page size as the threshold (XXX: tune)
  460          */
  461         if (pp->pr_size < palloc->pa_pagesz/16) {
  462                 /* Use the end of the page for the page header */
pp->pr_roflags |= PR_PHINPAGE;
pp->pr_phoffset = off = palloc->pa_pagesz -
ALIGN(sizeof(struct pool_item_header));
  466         } else {
  467                 /* The page header will be taken from our page header pool */
pp->pr_phoffset = 0;
off = palloc->pa_pagesz;
SPLAY_INIT(&pp->pr_phtree);
  471         }
  473         /*
  474          * Alignment is to take place at `ioff' within the item. This means
  475          * we must reserve up to `align - 1' bytes on the page to allow
  476          * appropriate positioning of each item.
  477          *
  478          * Silently enforce `0 <= ioff < align'.
  479          */
pp->pr_itemoffset = ioff = ioff % align;
pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
KASSERT(pp->pr_itemsperpage != 0);
  484         /*
  485          * Use the slack between the chunks and the page header
  486          * for "cache coloring".
  487          */
slack = off - pp->pr_itemsperpage * pp->pr_size;
pp->pr_maxcolor = (slack / align) * align;
pp->pr_curcolor = 0;
pp->pr_nget = 0;
pp->pr_nfail = 0;
pp->pr_nput = 0;
pp->pr_npagealloc = 0;
pp->pr_npagefree = 0;
pp->pr_hiwat = 0;
pp->pr_nidle = 0;
  500 #ifdef POOL_DIAGNOSTIC
  501         if (flags & PR_LOGGING) {
  502                 if (kmem_map == NULL ||
  503                     (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),
M_TEMP, M_NOWAIT)) == NULL)
pp->pr_roflags &= ~PR_LOGGING;
pp->pr_curlogentry = 0;
pp->pr_logsize = pool_logsize;
  508         }
  509 #endif
pp->pr_entered_file = NULL;
pp->pr_entered_line = 0;
simple_lock_init(&pp->pr_slock);
pp->pr_ipl = -1;
  518         /*
  519          * Initialize private page header pool and cache magazine pool if we
  520          * haven't done so yet.
  521          * XXX LOCKING.
  522          */
  523         if (phpool.pr_size == 0) {
pool_init(&phpool, sizeof(struct pool_item_header), 0, 0,
  525                     0, "phpool", NULL);
pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0,
  527                     0, "pcgpool", NULL);
  528         }
simple_lock_init(&pool_head_slock);
  532         /* Insert this into the list of all pools. */
simple_lock(&pool_head_slock);
TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
simple_unlock(&pool_head_slock);
  537         /* Insert into the list of pools using this allocator. */
simple_lock(&palloc->pa_slock);
TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
simple_unlock(&palloc->pa_slock);
  541 }
  543 #ifdef DIAGNOSTIC
  544 void
pool_setipl(struct pool *pp, int ipl)
  546 {
pp->pr_ipl = ipl;
  548 }
  549 #endif
  551 /*
  552  * Decommission a pool resource.
  553  */
  554 void
pool_destroy(struct pool *pp)
  556 {
  557         struct pool_item_header *ph;
  558         struct pool_cache *pc;
  560         /* Locking order: pool_allocator -> pool */
simple_lock(&pp->pr_alloc->pa_slock);
TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
simple_unlock(&pp->pr_alloc->pa_slock);
  565         /* Destroy all caches for this pool. */
  566         while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL)
pool_cache_destroy(pc);
  569 #ifdef DIAGNOSTIC
  570         if (pp->pr_nout != 0) {
pr_printlog(pp, NULL, printf);
panic("pool_destroy: pool busy: still out: %u",
pp->pr_nout);
  574         }
  575 #endif
  577         /* Remove all pages */
  578         while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
pr_rmpage(pp, ph, NULL);
KASSERT(LIST_EMPTY(&pp->pr_fullpages));
KASSERT(LIST_EMPTY(&pp->pr_partpages));
  583         /* Remove from global pool list */
simple_lock(&pool_head_slock);
TAILQ_REMOVE(&pool_head, pp, pr_poollist);
simple_unlock(&pool_head_slock);
  588 #ifdef POOL_DIAGNOSTIC
  589         if ((pp->pr_roflags & PR_LOGGING) != 0)
free(pp->pr_log, M_TEMP);
  591 #endif
  592 }
  594 static struct pool_item_header *
pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags)
  596 {
  597         struct pool_item_header *ph;
  598         int s;
LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0);
  602         if ((pp->pr_roflags & PR_PHINPAGE) != 0)
ph = (struct pool_item_header *) (storage + pp->pr_phoffset);
  604         else {
s = splhigh();
ph = pool_get(&phpool, flags);
splx(s);
  608         }
  610         return (ph);
  611 }
  613 /*
  614  * Grab an item from the pool; must be called at appropriate spl level
  615  */
  616 void *
  617 #ifdef POOL_DIAGNOSTIC
  618 _pool_get(struct pool *pp, int flags, const char *file, long line)
  619 #else
pool_get(struct pool *pp, int flags)
  621 #endif
  622 {
  623         struct pool_item *pi;
  624         struct pool_item_header *ph;
  625         void *v;
  627 #ifdef DIAGNOSTIC
  628         if ((flags & PR_WAITOK) != 0)
splassert(IPL_NONE);
  630         if (pp->pr_ipl != -1)
splassert(pp->pr_ipl);
  632         if (__predict_false(curproc == NULL && /* doing_shutdown == 0 && XXX*/
  633                             (flags & PR_WAITOK) != 0))
panic("pool_get: %s:must have NOWAIT", pp->pr_wchan);
  636 #ifdef LOCKDEBUG
  637         if (flags & PR_WAITOK)
simple_lock_only_held(NULL, "pool_get(PR_WAITOK)");
  639 #endif
  640 #endif /* DIAGNOSTIC */
  642 #ifdef MALLOC_DEBUG
  643         if (pp->pr_roflags & PR_DEBUG) {
  644                 void *addr;
addr = NULL;
debug_malloc(pp->pr_size, M_DEBUG,
  648                     (flags & PR_WAITOK) ? M_WAITOK : M_NOWAIT, &addr);
  649                 return (addr);
  650         }
  651 #endif
simple_lock(&pp->pr_slock);
pr_enter(pp, file, line);
startover:
  657         /*
  658          * Check to see if we've reached the hard limit.  If we have,
  659          * and we can wait, then wait until an item has been returned to
  660          * the pool.
  661          */
  662 #ifdef DIAGNOSTIC
  663         if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
pr_leave(pp);
simple_unlock(&pp->pr_slock);
panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
  667         }
  668 #endif
  669         if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
  670                 if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
  671                         /*
  672                          * XXX: A warning isn't logged in this case.  Should
  673                          * it be?
  674                          */
pp->pr_flags |= PR_WANTED;
pr_leave(pp);
ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
pr_enter(pp, file, line);
  679                         goto startover;
  680                 }
  682                 /*
  683                  * Log a message that the hard limit has been hit.
  684                  */
  685                 if (pp->pr_hardlimit_warning != NULL &&
ratecheck(&pp->pr_hardlimit_warning_last,
  687                               &pp->pr_hardlimit_ratecap))
log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
pp->pr_nfail++;
pr_leave(pp);
simple_unlock(&pp->pr_slock);
  694                 return (NULL);
  695         }
  697         /*
  698          * The convention we use is that if `curpage' is not NULL, then
  699          * it points at a non-empty bucket. In particular, `curpage'
  700          * never points at a page header which has PR_PHINPAGE set and
  701          * has no items in its bucket.
  702          */
  703         if ((ph = pp->pr_curpage) == NULL) {
  704 #ifdef DIAGNOSTIC
  705                 if (pp->pr_nitems != 0) {
simple_unlock(&pp->pr_slock);
printf("pool_get: %s: curpage NULL, nitems %u\n",
pp->pr_wchan, pp->pr_nitems);
panic("pool_get: nitems inconsistent");
  710                 }
  711 #endif
  713                 /*
  714                  * Call the back-end page allocator for more memory.
  715                  * Release the pool lock, as the back-end page allocator
  716                  * may block.
  717                  */
pr_leave(pp);
simple_unlock(&pp->pr_slock);
v = pool_allocator_alloc(pp, flags);
  721                 if (__predict_true(v != NULL))
ph = pool_alloc_item_header(pp, v, flags);
simple_lock(&pp->pr_slock);
pr_enter(pp, file, line);
  726                 if (__predict_false(v == NULL || ph == NULL)) {
  727                         if (v != NULL)
pool_allocator_free(pp, v);
  730                         /*
  731                          * We were unable to allocate a page or item
  732                          * header, but we released the lock during
  733                          * allocation, so perhaps items were freed
  734                          * back to the pool.  Check for this case.
  735                          */
  736                         if (pp->pr_curpage != NULL)
  737                                 goto startover;
  739                         if ((flags & PR_WAITOK) == 0) {
pp->pr_nfail++;
pr_leave(pp);
simple_unlock(&pp->pr_slock);
  743                                 return (NULL);
  744                         }
  746                         /*
  747                          * Wait for items to be returned to this pool.
  748                          *
  749                          * XXX: maybe we should wake up once a second and
  750                          * try again?
  751                          */
pp->pr_flags |= PR_WANTED;
  753                         /* PA_WANTED is already set on the allocator. */
pr_leave(pp);
ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
pr_enter(pp, file, line);
  757                         goto startover;
  758                 }
  760                 /* We have more memory; add it to the pool */
pool_prime_page(pp, v, ph);
pp->pr_npagealloc++;
  764                 /* Start the allocation process over. */
  765                 goto startover;
  766         }
  767         if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) {
pr_leave(pp);
simple_unlock(&pp->pr_slock);
panic("pool_get: %s: page empty", pp->pr_wchan);
  771         }
  772 #ifdef DIAGNOSTIC
  773         if (__predict_false(pp->pr_nitems == 0)) {
pr_leave(pp);
simple_unlock(&pp->pr_slock);
printf("pool_get: %s: items on itemlist, nitems %u\n",
pp->pr_wchan, pp->pr_nitems);
panic("pool_get: nitems inconsistent");
  779         }
  780 #endif
  782 #ifdef POOL_DIAGNOSTIC
pr_log(pp, v, PRLOG_GET, file, line);
  784 #endif
  786 #ifdef DIAGNOSTIC
  787         if (__predict_false(pi->pi_magic != PI_MAGIC)) {
pr_printlog(pp, pi, printf);
panic("pool_get(%s): free list modified: magic=%x; page %p;"
  790                        " item addr %p",
pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
  792         }
  793 #endif
  795         /*
  796          * Remove from item list.
  797          */
TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list);
pp->pr_nitems--;
pp->pr_nout++;
  801         if (ph->ph_nmissing == 0) {
  802 #ifdef DIAGNOSTIC
  803                 if (__predict_false(pp->pr_nidle == 0))
panic("pool_get: nidle inconsistent");
  805 #endif
pp->pr_nidle--;
  808                 /*
  809                  * This page was previously empty.  Move it to the list of
  810                  * partially-full pages.  This page is already curpage.
  811                  */
LIST_REMOVE(ph, ph_pagelist);
LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
  814         }
ph->ph_nmissing++;
  816         if (TAILQ_EMPTY(&ph->ph_itemlist)) {
  817 #ifdef DIAGNOSTIC
  818                 if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) {
pr_leave(pp);
simple_unlock(&pp->pr_slock);
panic("pool_get: %s: nmissing inconsistent",
pp->pr_wchan);
  823                 }
  824 #endif
  825                 /*
  826                  * This page is now full.  Move it to the full list
  827                  * and select a new current page.
  828                  */
LIST_REMOVE(ph, ph_pagelist);
LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
pool_update_curpage(pp);
  832         }
pp->pr_nget++;
  836         /*
  837          * If we have a low water mark and we are now below that low
  838          * water mark, add more items to the pool.
  839          */
  840         if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
  841                 /*
  842                  * XXX: Should we log a warning?  Should we set up a timeout
  843                  * to try again in a second or so?  The latter could break
  844                  * a caller's assumptions about interrupt protection, etc.
  845                  */
  846         }
pr_leave(pp);
simple_unlock(&pp->pr_slock);
  850         return (v);
  851 }
  853 /*
  854  * Internal version of pool_put().  Pool is already locked/entered.
  855  */
  856 void
pool_do_put(struct pool *pp, void *v)
  858 {
  859         struct pool_item *pi = v;
  860         struct pool_item_header *ph;
caddr_t page;
  863 #ifdef MALLOC_DEBUG
  864         if (pp->pr_roflags & PR_DEBUG) {
debug_free(v, M_DEBUG);
  866                 return;
  867         }
  868 #endif
LOCK_ASSERT(simple_lock_held(&pp->pr_slock));
page = (caddr_t)((vaddr_t)v & pp->pr_alloc->pa_pagemask);
  874 #ifdef DIAGNOSTIC
  875         if (pp->pr_ipl != -1)
splassert(pp->pr_ipl);
  878         if (__predict_false(pp->pr_nout == 0)) {
printf("pool %s: putting with none out\n",
pp->pr_wchan);
panic("pool_put");
  882         }
  883 #endif
  885         if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) {
pr_printlog(pp, NULL, printf);
panic("pool_put: %s: page header missing", pp->pr_wchan);
  888         }
  890 #ifdef LOCKDEBUG
  891         /*
  892          * Check if we're freeing a locked simple lock.
  893          */
simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size);
  895 #endif
  897         /*
  898          * Return to item list.
  899          */
  900 #ifdef DIAGNOSTIC
pi->pi_magic = PI_MAGIC;
  902 #endif
  903 #ifdef DEBUG
  904         {
  905                 int i, *ip = v;
  907                 for (i = 0; i < pp->pr_size / sizeof(int); i++) {
  908                         *ip++ = PI_MAGIC;
  909                 }
  910         }
  911 #endif
TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
ph->ph_nmissing--;
pp->pr_nput++;
pp->pr_nitems++;
pp->pr_nout--;
  919         /* Cancel "pool empty" condition if it exists */
  920         if (pp->pr_curpage == NULL)
pp->pr_curpage = ph;
  923         if (pp->pr_flags & PR_WANTED) {
pp->pr_flags &= ~PR_WANTED;
  925                 if (ph->ph_nmissing == 0)
pp->pr_nidle++;
wakeup(pp);
  928                 return;
  929         }
  931         /*
  932          * If this page is now empty, do one of two things:
  933          *
  934          *      (1) If we have more pages than the page high water mark,
  935          *          free the page back to the system.
  936          *
  937          *      (2) Otherwise, move the page to the empty page list.
  938          *
  939          * Either way, select a new current page (so we use a partially-full
  940          * page if one is available).
  941          */
  942         if (ph->ph_nmissing == 0) {
pp->pr_nidle++;
  944                 if (pp->pr_nidle > pp->pr_maxpages ||
  945                     (pp->pr_alloc->pa_flags & PA_WANT) != 0) {
pr_rmpage(pp, ph, NULL);
  947                 } else {
LIST_REMOVE(ph, ph_pagelist);
LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
  950                 }
pool_update_curpage(pp);
  952         }
  954         /*
  955          * If the page was previously completely full, move it to the
  956          * partially-full list and make it the current page.  The next
  957          * allocation will get the item from this page, instead of
  958          * further fragmenting the pool.
  959          */
  960         else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
LIST_REMOVE(ph, ph_pagelist);
LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
pp->pr_curpage = ph;
  964         }
  965 }
  967 /*
  968  * Return resource to the pool; must be called at appropriate spl level
  969  */
  970 #ifdef POOL_DIAGNOSTIC
  971 void
_pool_put(struct pool *pp, void *v, const char *file, long line)
  973 {
simple_lock(&pp->pr_slock);
pr_enter(pp, file, line);
pr_log(pp, v, PRLOG_PUT, file, line);
pool_do_put(pp, v);
pr_leave(pp);
simple_unlock(&pp->pr_slock);
  984 }
  985 #undef pool_put
  986 #endif /* POOL_DIAGNOSTIC */
  988 void
pool_put(struct pool *pp, void *v)
  990 {
simple_lock(&pp->pr_slock);
pool_do_put(pp, v);
simple_unlock(&pp->pr_slock);
  997 }
  999 #ifdef POOL_DIAGNOSTIC
 1000 #define         pool_put(h, v)  _pool_put((h), (v), __FILE__, __LINE__)
 1001 #endif
 1003 /*
 1004  * Add N items to the pool.
 1005  */
 1006 int
 1007 pool_prime(struct pool *pp, int n)
 1008 {
 1009         struct pool_item_header *ph;
caddr_t cp;
 1011         int newpages;
simple_lock(&pp->pr_slock);
newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
 1017         while (newpages-- > 0) {
simple_unlock(&pp->pr_slock);
cp = pool_allocator_alloc(pp, PR_NOWAIT);
 1020                 if (__predict_true(cp != NULL))
ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);
simple_lock(&pp->pr_slock);
 1024                 if (__predict_false(cp == NULL || ph == NULL)) {
 1025                         if (cp != NULL)
pool_allocator_free(pp, cp);
 1027                         break;
 1028                 }
pool_prime_page(pp, cp, ph);
pp->pr_npagealloc++;
pp->pr_minpages++;
 1033         }
 1035         if (pp->pr_minpages >= pp->pr_maxpages)
pp->pr_maxpages = pp->pr_minpages + 1;  /* XXX */
simple_unlock(&pp->pr_slock);
 1039         return (0);
 1040 }
 1042 /*
 1043  * Add a page worth of items to the pool.
 1044  *
 1045  * Note, we must be called with the pool descriptor LOCKED.
 1046  */
 1047 void
pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph)
 1049 {
 1050         struct pool_item *pi;
caddr_t cp = storage;
 1052         unsigned int align = pp->pr_align;
 1053         unsigned int ioff = pp->pr_itemoffset;
 1054         int n;
 1056 #ifdef DIAGNOSTIC
 1057         if (((u_long)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0)
panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);
 1059 #endif
 1061         /*
 1062          * Insert page header.
 1063          */
LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
TAILQ_INIT(&ph->ph_itemlist);
ph->ph_page = storage;
ph->ph_nmissing = 0;
 1068         if ((pp->pr_roflags & PR_PHINPAGE) == 0)
SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
pp->pr_nidle++;
 1073         /*
 1074          * Color this page.
 1075          */
cp = (caddr_t)(cp + pp->pr_curcolor);
 1077         if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
pp->pr_curcolor = 0;
 1080         /*
 1081          * Adjust storage to apply aligment to `pr_itemoffset' in each item.
 1082          */
 1083         if (ioff != 0)
cp = (caddr_t)(cp + (align - ioff));
 1086         /*
 1087          * Insert remaining chunks on the bucket list.
 1088          */
n = pp->pr_itemsperpage;
pp->pr_nitems += n;
 1092         while (n--) {
pi = (struct pool_item *)cp;
KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0);
 1097                 /* Insert on page list */
TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list);
 1099 #ifdef DIAGNOSTIC
pi->pi_magic = PI_MAGIC;
 1101 #endif
cp = (caddr_t)(cp + pp->pr_size);
 1103         }
 1105         /*
 1106          * If the pool was depleted, point at the new page.
 1107          */
 1108         if (pp->pr_curpage == NULL)
pp->pr_curpage = ph;
 1111         if (++pp->pr_npages > pp->pr_hiwat)
pp->pr_hiwat = pp->pr_npages;
 1113 }
 1115 /*
 1116  * Used by pool_get() when nitems drops below the low water mark.  This
 1117  * is used to catch up pr_nitems with the low water mark.
 1118  *
 1119  * Note 1, we never wait for memory here, we let the caller decide what to do.
 1120  *
 1121  * Note 2, we must be called with the pool already locked, and we return
 1122  * with it locked.
 1123  */
 1124 int
pool_catchup(struct pool *pp)
 1126 {
 1127         struct pool_item_header *ph;
caddr_t cp;
 1129         int error = 0;
 1131         while (POOL_NEEDS_CATCHUP(pp)) {
 1132                 /*
 1133                  * Call the page back-end allocator for more memory.
 1134                  *
 1135                  * XXX: We never wait, so should we bother unlocking
 1136                  * the pool descriptor?
 1137                  */
simple_unlock(&pp->pr_slock);
cp = pool_allocator_alloc(pp, PR_NOWAIT);
 1140                 if (__predict_true(cp != NULL))
ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);
simple_lock(&pp->pr_slock);
 1143                 if (__predict_false(cp == NULL || ph == NULL)) {
 1144                         if (cp != NULL)
pool_allocator_free(pp, cp);
error = ENOMEM;
 1147                         break;
 1148                 }
pool_prime_page(pp, cp, ph);
pp->pr_npagealloc++;
 1151         }
 1153         return (error);
 1154 }
 1156 void
pool_update_curpage(struct pool *pp)
 1158 {
pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
 1161         if (pp->pr_curpage == NULL) {
pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
 1163         }
 1164 }
 1166 void
pool_setlowat(struct pool *pp, int n)
 1168 {
simple_lock(&pp->pr_slock);
pp->pr_minitems = n;
pp->pr_minpages = (n == 0)
 1174                 ? 0
 1175                 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
 1177         /* Make sure we're caught up with the newly-set low water mark. */
 1178         if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
 1179                 /*
 1180                  * XXX: Should we log a warning?  Should we set up a timeout
 1181                  * to try again in a second or so?  The latter could break
 1182                  * a caller's assumptions about interrupt protection, etc.
 1183                  */
 1184         }
simple_unlock(&pp->pr_slock);
 1187 }
 1189 void
pool_sethiwat(struct pool *pp, int n)
 1191 {
simple_lock(&pp->pr_slock);
pp->pr_maxpages = (n == 0)
 1196                 ? 0
 1197                 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
simple_unlock(&pp->pr_slock);
 1200 }
 1202 int
pool_sethardlimit(struct pool *pp, unsigned n, const char *warnmess, int ratecap)
 1204 {
 1205         int error = 0;
simple_lock(&pp->pr_slock);
 1209         if (n < pp->pr_nout) {
error = EINVAL;
 1211                 goto done;
 1212         }
pp->pr_hardlimit = n;
pp->pr_hardlimit_warning = warnmess;
pp->pr_hardlimit_ratecap.tv_sec = ratecap;
pp->pr_hardlimit_warning_last.tv_sec = 0;
pp->pr_hardlimit_warning_last.tv_usec = 0;
 1220         /*
 1221          * In-line version of pool_sethiwat(), because we don't want to
 1222          * release the lock.
 1223          */
pp->pr_maxpages = (n == 0 || n == UINT_MAX)
 1225                 ? n
 1226                 : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
done:
simple_unlock(&pp->pr_slock);
 1231         return (error);
 1232 }
 1234 /*
 1235  * Release all complete pages that have not been used recently.
 1236  *
 1237  * Returns non-zero if any pages have been reclaimed.
 1238  */
 1239 int
 1240 #ifdef POOL_DIAGNOSTIC
 1241 _pool_reclaim(struct pool *pp, const char *file, long line)
 1242 #else
pool_reclaim(struct pool *pp)
 1244 #endif
 1245 {
 1246         struct pool_item_header *ph, *phnext;
 1247         struct pool_cache *pc;
 1248         struct pool_pagelist pq;
 1249         int s;
 1251         if (simple_lock_try(&pp->pr_slock) == 0)
 1252                 return (0);
pr_enter(pp, file, line);
LIST_INIT(&pq);
 1257         /*
 1258          * Reclaim items from the pool's caches.
 1259          */
TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist)
pool_cache_reclaim(pc);
 1263         for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
phnext = LIST_NEXT(ph, ph_pagelist);
 1266                 /* Check our minimum page claim */
 1267                 if (pp->pr_npages <= pp->pr_minpages)
 1268                         break;
KASSERT(ph->ph_nmissing == 0);
 1272                 /*
 1273                  * If freeing this page would put us below
 1274                  * the low water mark, stop now.
 1275                  */
 1276                 if ((pp->pr_nitems - pp->pr_itemsperpage) <
pp->pr_minitems)
 1278                         break;
pr_rmpage(pp, ph, &pq);
 1281         }
pr_leave(pp);
simple_unlock(&pp->pr_slock);
 1285         if (LIST_EMPTY(&pq))
 1286                 return (0);
 1287         while ((ph = LIST_FIRST(&pq)) != NULL) {
LIST_REMOVE(ph, ph_pagelist);
pool_allocator_free(pp, ph->ph_page);
 1290                 if (pp->pr_roflags & PR_PHINPAGE) {
 1291                         continue;
 1292                 }
SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
s = splhigh();
pool_put(&phpool, ph);
splx(s);
 1297         }
 1299         return (1);
 1300 }
 1302 #ifdef DDB
 1303 #include <machine/db_machdep.h>
 1304 #include <ddb/db_interface.h>
 1305 #include <ddb/db_output.h>
 1307 /*
 1308  * Diagnostic helpers.
 1309  */
 1310 void
pool_printit(struct pool *pp, const char *modif, int (*pr)(const char *, ...))
 1312 {
 1313         int s;
s = splvm();
 1316         if (simple_lock_try(&pp->pr_slock) == 0) {
pr("pool %s is locked; try again later\n",
pp->pr_wchan);
splx(s);
 1320                 return;
 1321         }
pool_print1(pp, modif, pr);
simple_unlock(&pp->pr_slock);
splx(s);
 1325 }
 1327 void
pool_print_pagelist(struct pool_pagelist *pl, int (*pr)(const char *, ...))
 1329 {
 1330         struct pool_item_header *ph;
 1331 #ifdef DIAGNOSTIC
 1332         struct pool_item *pi;
 1333 #endif
LIST_FOREACH(ph, pl, ph_pagelist) {
 1336                 (*pr)("\t\tpage %p, nmissing %d\n",
ph->ph_page, ph->ph_nmissing);
 1338 #ifdef DIAGNOSTIC
TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) {
 1340                         if (pi->pi_magic != PI_MAGIC) {
 1341                                 (*pr)("\t\t\titem %p, magic 0x%x\n",
pi, pi->pi_magic);
 1343                         }
 1344                 }
 1345 #endif
 1346         }
 1347 }
 1349 void
pool_print1(struct pool *pp, const char *modif, int (*pr)(const char *, ...))
 1351 {
 1352         struct pool_item_header *ph;
 1353         struct pool_cache *pc;
 1354         struct pool_cache_group *pcg;
 1355         int i, print_log = 0, print_pagelist = 0, print_cache = 0;
 1356         char c;
 1358         while ((c = *modif++) != '\0') {
 1359                 if (c == 'l')
print_log = 1;
 1361                 if (c == 'p')
print_pagelist = 1;
 1363                 if (c == 'c')
print_cache = 1;
modif++;
 1366         }
 1368         (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n",
pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,
pp->pr_roflags);
 1371         (*pr)("\talloc %p\n", pp->pr_alloc);
 1372         (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
 1374         (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
 1377         (*pr)("\n\tnget %lu, nfail %lu, nput %lu\n",
pp->pr_nget, pp->pr_nfail, pp->pr_nput);
 1379         (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
 1382         if (print_pagelist == 0)
 1383                 goto skip_pagelist;
 1385         if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
 1386                 (*pr)("\n\tempty page list:\n");
pool_print_pagelist(&pp->pr_emptypages, pr);
 1388         if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
 1389                 (*pr)("\n\tfull page list:\n");
pool_print_pagelist(&pp->pr_fullpages, pr);
 1391         if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
 1392                 (*pr)("\n\tpartial-page list:\n");
pool_print_pagelist(&pp->pr_partpages, pr);
 1395         if (pp->pr_curpage == NULL)
 1396                 (*pr)("\tno current page\n");
 1397         else
 1398                 (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
skip_pagelist:
 1401         if (print_log == 0)
 1402                 goto skip_log;
 1404         (*pr)("\n");
 1405         if ((pp->pr_roflags & PR_LOGGING) == 0)
 1406                 (*pr)("\tno log\n");
 1407         else
pr_printlog(pp, NULL, pr);
skip_log:
 1411         if (print_cache == 0)
 1412                 goto skip_cache;
TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) {
 1415                 (*pr)("\tcache %p: allocfrom %p freeto %p\n", pc,
pc->pc_allocfrom, pc->pc_freeto);
 1417                 (*pr)("\t    hits %lu misses %lu ngroups %lu nitems %lu\n",
pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);
TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
 1420                         (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);
 1421                         for (i = 0; i < PCG_NOBJECTS; i++)
 1422                                 (*pr)("\t\t\t%p\n", pcg->pcg_objects[i]);
 1423                 }
 1424         }
skip_cache:
pr_enter_check(pp, pr);
 1428 }
 1430 void
db_show_all_pools(db_expr_t expr, int haddr, db_expr_t count, char *modif)
 1432 {
 1433         struct pool *pp;
 1434         char maxp[16];
 1435         int ovflw;
 1436         char mode;
mode = modif[0];
 1439         if (mode != '\0' && mode != 'a') {
db_printf("usage: show all pools [/a]\n");
 1441                 return;
 1442         }
 1444         if (mode == '\0')
db_printf("%-10s%4s%9s%5s%9s%6s%6s%6s%6s%6s%6s%5s\n",
 1446                     "Name",
 1447                     "Size",
 1448                     "Requests",
 1449                     "Fail",
 1450                     "Releases",
 1451                     "Pgreq",
 1452                     "Pgrel",
 1453                     "Npage",
 1454                     "Hiwat",
 1455                     "Minpg",
 1456                     "Maxpg",
 1457                     "Idle");
 1458         else
db_printf("%-10s %18s %18s\n",
 1460                     "Name", "Address", "Allocator");
TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
 1463                 if (mode == 'a') {
db_printf("%-10s %18p %18p\n", pp->pr_wchan, pp,
pp->pr_alloc);
 1466                         continue;
 1467                 }
 1469                 if (!pp->pr_nget)
 1470                         continue;
 1472                 if (pp->pr_maxpages == UINT_MAX)
snprintf(maxp, sizeof maxp, "inf");
 1474                 else
snprintf(maxp, sizeof maxp, "%u", pp->pr_maxpages);
 1477 #define PRWORD(ovflw, fmt, width, fixed, val) do {      \
 1478         (ovflw) += db_printf((fmt),                     \
 1479             (width) - (fixed) - (ovflw) > 0 ?           \
 1480             (width) - (fixed) - (ovflw) : 0,            \
 1481             (val)) - (width);                           \
 1482         if ((ovflw) < 0)                                \
 1483                 (ovflw) = 0;                            \
 1484 } while (/* CONSTCOND */0)
ovflw = 0;
PRWORD(ovflw, "%-*s", 10, 0, pp->pr_wchan);
PRWORD(ovflw, " %*u", 4, 1, pp->pr_size);
PRWORD(ovflw, " %*lu", 9, 1, pp->pr_nget);
PRWORD(ovflw, " %*lu", 5, 1, pp->pr_nfail);
PRWORD(ovflw, " %*lu", 9, 1, pp->pr_nput);
PRWORD(ovflw, " %*lu", 6, 1, pp->pr_npagealloc);
PRWORD(ovflw, " %*lu", 6, 1, pp->pr_npagefree);
PRWORD(ovflw, " %*d", 6, 1, pp->pr_npages);
PRWORD(ovflw, " %*d", 6, 1, pp->pr_hiwat);
PRWORD(ovflw, " %*d", 6, 1, pp->pr_minpages);
PRWORD(ovflw, " %*s", 6, 1, maxp);
PRWORD(ovflw, " %*lu\n", 5, 1, pp->pr_nidle);
 1499         }
 1500 }
 1502 int
pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
 1504 {
 1505         struct pool_item *pi;
caddr_t page;
 1507         int n;
page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask);
 1510         if (page != ph->ph_page &&
 1511             (pp->pr_roflags & PR_PHINPAGE) != 0) {
 1512                 if (label != NULL)
printf("%s: ", label);
printf("pool(%p:%s): page inconsistency: page %p;"
 1515                        " at page head addr %p (p %p)\n", pp,
pp->pr_wchan, ph->ph_page,
ph, page);
 1518                 return 1;
 1519         }
 1521         for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0;
pi != NULL;
pi = TAILQ_NEXT(pi,pi_list), n++) {
 1525 #ifdef DIAGNOSTIC
 1526                 if (pi->pi_magic != PI_MAGIC) {
 1527                         if (label != NULL)
printf("%s: ", label);
printf("pool(%s): free list modified: magic=%x;"
 1530                                " page %p; item ordinal %d;"
 1531                                " addr %p (p %p)\n",
pp->pr_wchan, pi->pi_magic, ph->ph_page,
n, pi, page);
panic("pool");
 1535                 }
 1536 #endif
page =
 1538                     (caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask);
 1539                 if (page == ph->ph_page)
 1540                         continue;
 1542                 if (label != NULL)
printf("%s: ", label);
printf("pool(%p:%s): page inconsistency: page %p;"
 1545                        " item ordinal %d; addr %p (p %p)\n", pp,
pp->pr_wchan, ph->ph_page,
n, pi, page);
 1548                 return 1;
 1549         }
 1550         return 0;
 1551 }
 1553 int
 1554 pool_chk(struct pool *pp, const char *label)
 1555 {
 1556         struct pool_item_header *ph;
 1557         int r = 0;
simple_lock(&pp->pr_slock);
LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
r = pool_chk_page(pp, label, ph);
 1562                 if (r) {
 1563                         goto out;
 1564                 }
 1565         }
LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
r = pool_chk_page(pp, label, ph);
 1568                 if (r) {
 1569                         goto out;
 1570                 }
 1571         }
LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
r = pool_chk_page(pp, label, ph);
 1574                 if (r) {
 1575                         goto out;
 1576                 }
 1577         }
out:
simple_unlock(&pp->pr_slock);
 1581         return (r);
 1582 }
 1583 #endif
 1585 /*
 1586  * pool_cache_init:
 1587  *
 1588  *      Initialize a pool cache.
 1589  *
 1590  *      NOTE: If the pool must be protected from interrupts, we expect
 1591  *      to be called at the appropriate interrupt priority level.
 1592  */
 1593 void
 1594 pool_cache_init(struct pool_cache *pc, struct pool *pp,
 1595     int (*ctor)(void *, void *, int),
 1596     void (*dtor)(void *, void *),
 1597     void *arg)
 1598 {
TAILQ_INIT(&pc->pc_grouplist);
simple_lock_init(&pc->pc_slock);
pc->pc_allocfrom = NULL;
pc->pc_freeto = NULL;
pc->pc_pool = pp;
pc->pc_ctor = ctor;
pc->pc_dtor = dtor;
pc->pc_arg  = arg;
pc->pc_hits   = 0;
pc->pc_misses = 0;
pc->pc_ngroups = 0;
pc->pc_nitems = 0;
simple_lock(&pp->pr_slock);
TAILQ_INSERT_TAIL(&pp->pr_cachelist, pc, pc_poollist);
simple_unlock(&pp->pr_slock);
 1621 }
 1623 /*
 1624  * pool_cache_destroy:
 1625  *
 1626  *      Destroy a pool cache.
 1627  */
 1628 void
pool_cache_destroy(struct pool_cache *pc)
 1630 {
 1631         struct pool *pp = pc->pc_pool;
 1633         /* First, invalidate the entire cache. */
pool_cache_invalidate(pc);
 1636         /* ...and remove it from the pool's cache list. */
simple_lock(&pp->pr_slock);
TAILQ_REMOVE(&pp->pr_cachelist, pc, pc_poollist);
simple_unlock(&pp->pr_slock);
 1640 }
 1642 static __inline void *
pcg_get(struct pool_cache_group *pcg)
 1644 {
 1645         void *object;
u_int idx;
KASSERT(pcg->pcg_avail <= PCG_NOBJECTS);
KASSERT(pcg->pcg_avail != 0);
idx = --pcg->pcg_avail;
KASSERT(pcg->pcg_objects[idx] != NULL);
object = pcg->pcg_objects[idx];
pcg->pcg_objects[idx] = NULL;
 1656         return (object);
 1657 }
 1659 static __inline void
pcg_put(struct pool_cache_group *pcg, void *object)
 1661 {
u_int idx;
KASSERT(pcg->pcg_avail < PCG_NOBJECTS);
idx = pcg->pcg_avail++;
KASSERT(pcg->pcg_objects[idx] == NULL);
pcg->pcg_objects[idx] = object;
 1669 }
 1671 /*
 1672  * pool_cache_get:
 1673  *
 1674  *      Get an object from a pool cache.
 1675  */
 1676 void *
 1677 pool_cache_get(struct pool_cache *pc, int flags)
 1678 {
 1679         struct pool_cache_group *pcg;
 1680         void *object;
 1682 #ifdef LOCKDEBUG
 1683         if (flags & PR_WAITOK)
simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)");
 1685 #endif
simple_lock(&pc->pc_slock);
 1689         if ((pcg = pc->pc_allocfrom) == NULL) {
TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
 1691                         if (pcg->pcg_avail != 0) {
pc->pc_allocfrom = pcg;
 1693                                 goto have_group;
 1694                         }
 1695                 }
 1697                 /*
 1698                  * No groups with any available objects.  Allocate
 1699                  * a new object, construct it, and return it to
 1700                  * the caller.  We will allocate a group, if necessary,
 1701                  * when the object is freed back to the cache.
 1702                  */
pc->pc_misses++;
simple_unlock(&pc->pc_slock);
object = pool_get(pc->pc_pool, flags);
 1706                 if (object != NULL && pc->pc_ctor != NULL) {
 1707                         if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {
pool_put(pc->pc_pool, object);
 1709                                 return (NULL);
 1710                         }
 1711                 }
 1712                 return (object);
 1713         }
have_group:
pc->pc_hits++;
pc->pc_nitems--;
object = pcg_get(pcg);
 1720         if (pcg->pcg_avail == 0)
pc->pc_allocfrom = NULL;
simple_unlock(&pc->pc_slock);
 1725         return (object);
 1726 }
 1728 /*
 1729  * pool_cache_put:
 1730  *
 1731  *      Put an object back to the pool cache.
 1732  */
 1733 void
 1734 pool_cache_put(struct pool_cache *pc, void *object)
 1735 {
 1736         struct pool_cache_group *pcg;
 1737         int s;
simple_lock(&pc->pc_slock);
 1741         if ((pcg = pc->pc_freeto) == NULL) {
TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
 1743                         if (pcg->pcg_avail != PCG_NOBJECTS) {
pc->pc_freeto = pcg;
 1745                                 goto have_group;
 1746                         }
 1747                 }
 1749                 /*
 1750                  * No empty groups to free the object to.  Attempt to
 1751                  * allocate one.
 1752                  */
simple_unlock(&pc->pc_slock);
s = splvm();
pcg = pool_get(&pcgpool, PR_NOWAIT);
splx(s);
 1757                 if (pcg != NULL) {
memset(pcg, 0, sizeof(*pcg));
simple_lock(&pc->pc_slock);
pc->pc_ngroups++;
TAILQ_INSERT_TAIL(&pc->pc_grouplist, pcg, pcg_list);
 1762                         if (pc->pc_freeto == NULL)
pc->pc_freeto = pcg;
 1764                         goto have_group;
 1765                 }
 1767                 /*
 1768                  * Unable to allocate a cache group; destruct the object
 1769                  * and free it back to the pool.
 1770                  */
pool_cache_destruct_object(pc, object);
 1772                 return;
 1773         }
have_group:
pc->pc_nitems++;
pcg_put(pcg, object);
 1779         if (pcg->pcg_avail == PCG_NOBJECTS)
pc->pc_freeto = NULL;
simple_unlock(&pc->pc_slock);
 1783 }
 1785 /*
 1786  * pool_cache_destruct_object:
 1787  *
 1788  *      Force destruction of an object and its release back into
 1789  *      the pool.
 1790  */
 1791 void
pool_cache_destruct_object(struct pool_cache *pc, void *object)
 1793 {
 1795         if (pc->pc_dtor != NULL)
 1796                 (*pc->pc_dtor)(pc->pc_arg, object);
pool_put(pc->pc_pool, object);
 1798 }
 1800 /*
 1801  * pool_cache_do_invalidate:
 1802  *
 1803  *      This internal function implements pool_cache_invalidate() and
 1804  *      pool_cache_reclaim().
 1805  */
 1806 void
pool_cache_do_invalidate(struct pool_cache *pc, int free_groups,
 1808     void (*putit)(struct pool *, void *))
 1809 {
 1810         struct pool_cache_group *pcg, *npcg;
 1811         void *object;
 1812         int s;
 1814         for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;
pcg = npcg) {
npcg = TAILQ_NEXT(pcg, pcg_list);
 1817                 while (pcg->pcg_avail != 0) {
pc->pc_nitems--;
object = pcg_get(pcg);
 1820                         if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg)
pc->pc_allocfrom = NULL;
 1822                         if (pc->pc_dtor != NULL)
 1823                                 (*pc->pc_dtor)(pc->pc_arg, object);
 1824                         (*putit)(pc->pc_pool, object);
 1825                 }
 1826                 if (free_groups) {
pc->pc_ngroups--;
TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list);
 1829                         if (pc->pc_freeto == pcg)
pc->pc_freeto = NULL;
s = splvm();
pool_put(&pcgpool, pcg);
splx(s);
 1834                 }
 1835         }
 1836 }
 1838 /*
 1839  * pool_cache_invalidate:
 1840  *
 1841  *      Invalidate a pool cache (destruct and release all of the
 1842  *      cached objects).
 1843  */
 1844 void
pool_cache_invalidate(struct pool_cache *pc)
 1846 {
simple_lock(&pc->pc_slock);
pool_cache_do_invalidate(pc, 0, pool_put);
simple_unlock(&pc->pc_slock);
 1851 }
 1853 /*
 1854  * pool_cache_reclaim:
 1855  *
 1856  *      Reclaim a pool cache for pool_reclaim().
 1857  */
 1858 void
pool_cache_reclaim(struct pool_cache *pc)
 1860 {
simple_lock(&pc->pc_slock);
pool_cache_do_invalidate(pc, 1, pool_do_put);
simple_unlock(&pc->pc_slock);
 1865 }
 1867 /*
 1868  * We have three different sysctls.
 1869  * kern.pool.npools - the number of pools.
 1870  * kern.pool.pool.<pool#> - the pool struct for the pool#.
 1871  * kern.pool.name.<pool#> - the name for pool#.
 1872  */
 1873 int
sysctl_dopool(int *name, u_int namelen, char *where, size_t *sizep)
 1875 {
 1876         struct pool *pp, *foundpool = NULL;
size_t buflen = where != NULL ? *sizep : 0;
 1878         int npools = 0, s;
 1879         unsigned int lookfor;
size_t len;
 1882         switch (*name) {
 1883         case KERN_POOL_NPOOLS:
 1884                 if (namelen != 1 || buflen != sizeof(int))
 1885                         return (EINVAL);
lookfor = 0;
 1887                 break;
 1888         case KERN_POOL_NAME:
 1889                 if (namelen != 2 || buflen < 1)
 1890                         return (EINVAL);
lookfor = name[1];
 1892                 break;
 1893         case KERN_POOL_POOL:
 1894                 if (namelen != 2 || buflen != sizeof(struct pool))
 1895                         return (EINVAL);
lookfor = name[1];
 1897                 break;
 1898         default:
 1899                 return (EINVAL);
 1900         }
s = splvm();
simple_lock(&pool_head_slock);
TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
npools++;
 1907                 if (lookfor == pp->pr_serial) {
foundpool = pp;
 1909                         break;
 1910                 }
 1911         }
simple_unlock(&pool_head_slock);
splx(s);
 1916         if (*name != KERN_POOL_NPOOLS && foundpool == NULL)
 1917                 return (ENOENT);
 1919         switch (*name) {
 1920         case KERN_POOL_NPOOLS:
 1921                 return copyout(&npools, where, buflen);
 1922         case KERN_POOL_NAME:
len = strlen(foundpool->pr_wchan) + 1;
 1924                 if (*sizep < len)
 1925                         return (ENOMEM);
 1926                 *sizep = len;
 1927                 return copyout(foundpool->pr_wchan, where, len);
 1928         case KERN_POOL_POOL:
 1929                 return copyout(foundpool, where, buflen);
 1930         }
 1931         /* NOTREACHED */
 1932         return (0); /* XXX - Stupid gcc */
 1933 }
 1935 /*
 1936  * Pool backend allocators.
 1937  *
 1938  * Each pool has a backend allocator that handles allocation, deallocation
 1939  */
 1940 void    *pool_page_alloc_oldnointr(struct pool *, int);
 1941 void    pool_page_free_oldnointr(struct pool *, void *);
 1942 void    *pool_page_alloc(struct pool *, int);
 1943 void    pool_page_free(struct pool *, void *);
 1945 /* previous nointr.  handles large allocations safely */
 1946 struct pool_allocator pool_allocator_oldnointr = {
pool_page_alloc_oldnointr, pool_page_free_oldnointr, 0,
 1948 };
 1949 /* safe for interrupts, name preserved for compat
 1950  * this is the default allocator */
 1951 struct pool_allocator pool_allocator_nointr = {
pool_page_alloc, pool_page_free, 0,
 1953 };
 1955 /*
 1956  * XXX - we have at least three different resources for the same allocation
 1957  *  and each resource can be depleted. First we have the ready elements in
 1958  *  the pool. Then we have the resource (typically a vm_map) for this
 1959  *  allocator, then we have physical memory. Waiting for any of these can
 1960  *  be unnecessary when any other is freed, but the kernel doesn't support
 1961  *  sleeping on multiple addresses, so we have to fake. The caller sleeps on
 1962  *  the pool (so that we can be awakened when an item is returned to the pool),
 1963  *  but we set PA_WANT on the allocator. When a page is returned to
 1964  *  the allocator and PA_WANT is set pool_allocator_free will wakeup all
 1965  *  sleeping pools belonging to this allocator. (XXX - thundering herd).
 1966  *  We also wake up the allocator in case someone without a pool (malloc)
 1967  *  is sleeping waiting for this allocator.
 1968  */
 1970 void *
pool_allocator_alloc(struct pool *pp, int flags)
 1972 {
 1974         return (pp->pr_alloc->pa_alloc(pp, flags));
 1975 }
 1977 void
pool_allocator_free(struct pool *pp, void *v)
 1979 {
 1980         struct pool_allocator *pa = pp->pr_alloc;
 1981         int s;
 1983         (*pa->pa_free)(pp, v);
s = splvm();
simple_lock(&pa->pa_slock);
 1987         if ((pa->pa_flags & PA_WANT) == 0) {
simple_unlock(&pa->pa_slock);
splx(s);
 1990                 return;
 1991         }
TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) {
simple_lock(&pp->pr_slock);
 1995                 if ((pp->pr_flags & PR_WANTED) != 0) {
pp->pr_flags &= ~PR_WANTED;
wakeup(pp);
 1998                 }
simple_unlock(&pp->pr_slock);
 2000         }
pa->pa_flags &= ~PA_WANT;
simple_unlock(&pa->pa_slock);
splx(s);
 2004 }
 2006 void *
pool_page_alloc(struct pool *pp, int flags)
 2008 {
boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
 2011         return (uvm_km_getpage(waitok));
 2012 }
 2014 void
pool_page_free(struct pool *pp, void *v)
 2016 {
uvm_km_putpage(v);
 2019 }
 2021 void *
pool_page_alloc_oldnointr(struct pool *pp, int flags)
 2023 {
boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
splassert(IPL_NONE);
 2028         return ((void *)uvm_km_alloc_poolpage1(kernel_map, uvm.kernel_object,
waitok));
 2030 }
 2032 void
pool_page_free_oldnointr(struct pool *pp, void *v)
 2034 {
splassert(IPL_NONE);
uvm_km_free_poolpage1(kernel_map, (vaddr_t)v);
 2038 }