root/dev/pci/safe.c

DEFINITIONS

1. safe_probe
2. safe_attach
3. safe_process
4. safe_reset_board
5. safe_init_board
6. safe_init_pciregs
7. safe_dma_malloc
8. safe_dma_free
9. safe_rng_init
10. safe_rng_read
11. safe_rng
12. safe_newsession
13. safe_freesession
14. safe_dmamap_aligned
15. safe_cleanchip
16. safe_free_entry
17. safe_feed
18. safe_dmamap_uniform
19. safe_mcopy
20. safe_callback
21. safe_intr
22. safe_kfind
23. safe_kprocess
24. safe_kstart
25. safe_ksigbits
26. safe_kfeed
27. safe_kpoll
28. safe_kload_reg
29. safe_dump_dmastatus
30. safe_dump_intrstate
31. safe_dump_ringstate
32. safe_dump_request
33. safe_dump_ring

    1 /*      $OpenBSD: safe.c,v 1.21 2007/02/28 22:16:55 deraadt Exp $       */
    3 /*-
    4  * Copyright (c) 2003 Sam Leffler, Errno Consulting
    5  * Copyright (c) 2003 Global Technology Associates, Inc.
    6  * All rights reserved.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  * $FreeBSD: /repoman/r/ncvs/src/sys/dev/safe/safe.c,v 1.1 2003/07/21 21:46:07 sam Exp $
   30  */
   32 #include <sys/cdefs.h>
   34 /*
   35  * SafeNet SafeXcel-1141 hardware crypto accelerator
   36  */
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/proc.h>
   40 #include <sys/errno.h>
   41 #include <sys/malloc.h>
   42 #include <sys/kernel.h>
   43 #include <sys/mbuf.h>
   44 #include <sys/device.h>
   45 #include <sys/timeout.h>
   47 #include <machine/bus.h>
   49 #include <crypto/md5.h>
   50 #include <crypto/sha1.h>
   51 #include <crypto/cryptodev.h>
   52 #include <crypto/cryptosoft.h>
   53 #include <dev/rndvar.h>
   55 #include <dev/pci/pcivar.h>
   56 #include <dev/pci/pcireg.h>
   57 #include <dev/pci/pcidevs.h>
   59 #include <dev/pci/safereg.h>
   60 #include <dev/pci/safevar.h>
   62 #ifndef bswap32
   63 #define bswap32 NTOHL
   64 #endif
   66 #define KASSERT_X(x,y)
   68 /*
   69  * Prototypes and count for the pci_device structure
   70  */
   71 int safe_probe(struct device *, void *, void *);
   72 void safe_attach(struct device *, struct device *, void *);
   74 struct cfattach safe_ca = {
   75         sizeof(struct safe_softc), safe_probe, safe_attach
   76 };
   78 struct cfdriver safe_cd = {
   79         0, "safe", DV_DULL
   80 };
   82 int safe_intr(void *);
   83 int safe_newsession(u_int32_t *, struct cryptoini *);
   84 int safe_freesession(u_int64_t);
   85 int safe_process(struct cryptop *);
   86 int safe_kprocess(struct cryptkop *);
   87 int safe_kstart(struct safe_softc *);
   88 void safe_kload_reg(struct safe_softc *, u_int32_t, u_int32_t,
   89     struct crparam *);
   90 struct safe_softc *safe_kfind(struct cryptkop *);
   91 void safe_kpoll(void *);
   92 void safe_kfeed(struct safe_softc *);
   93 int safe_ksigbits(struct crparam *cr);
   94 void safe_callback(struct safe_softc *, struct safe_ringentry *);
   95 void safe_feed(struct safe_softc *, struct safe_ringentry *);
   96 void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
   97 void safe_rng_init(struct safe_softc *);
   98 void safe_rng(void *);
   99 int safe_dma_malloc(struct safe_softc *, bus_size_t,
  100                 struct safe_dma_alloc *, int);
  101 #define safe_dma_sync(_sc, _dma, _flags) \
bus_dmamap_sync((_sc)->sc_dmat, (_dma)->dma_map, 0,             \
  103             (_dma)->dma_map->dm_mapsize, (_flags))
  104 void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
  105 int safe_dmamap_aligned(const struct safe_operand *);
  106 int safe_dmamap_uniform(const struct safe_operand *);
  108 void safe_reset_board(struct safe_softc *);
  109 void safe_init_board(struct safe_softc *);
  110 void safe_init_pciregs(struct safe_softc *);
  111 void safe_cleanchip(struct safe_softc *);
  112 __inline u_int32_t safe_rng_read(struct safe_softc *);
  114 int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
  116 #ifdef SAFE_DEBUG
  117 int safe_debug;
  118 #define DPRINTF(_x)     if (safe_debug) printf _x
  120 void safe_dump_dmastatus(struct safe_softc *, const char *);
  121 void safe_dump_intrstate(struct safe_softc *, const char *);
  122 void safe_dump_ringstate(struct safe_softc *, const char *);
  123 void safe_dump_request(struct safe_softc *, const char *,
  124     struct safe_ringentry *);
  125 void safe_dump_ring(struct safe_softc *sc, const char *tag);
  126 #else
  127 #define DPRINTF(_x)
  128 #endif
  130 #define READ_REG(sc,r) \
bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
  133 #define WRITE_REG(sc,reg,val) \
bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
  136 struct safe_stats safestats;
  138 int safe_rnginterval = 1;               /* poll once a second */
  139 int safe_rngbufsize = 16;               /* 64 bytes each poll  */
  140 int safe_rngmaxalarm = 8;               /* max alarms before reset */
  142 int
safe_probe(struct device *parent, void *match, void *aux)
  144 {
  145         struct pci_attach_args *pa = aux;
  147         if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_SAFENET &&
PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_SAFENET_SAFEXCEL)
  149                 return (1);
  150         return (0);
  151 }
  153 void
safe_attach(struct device *parent, struct device *self, void *aux)
  155 {
  156         struct safe_softc *sc = (struct safe_softc *)self;
  157         struct pci_attach_args *pa = aux;
pci_intr_handle_t ih;
  159         const char *intrstr = NULL;
bus_size_t iosize;
bus_addr_t raddr;
u_int32_t devinfo;
  163         int algs[CRYPTO_ALGORITHM_MAX + 1], i;
  165         /* XXX handle power management */
SIMPLEQ_INIT(&sc->sc_pkq);
sc->sc_dmat = pa->pa_dmat;
  170         /* 
  171          * Setup memory-mapping of PCI registers.
  172          */
  173         if (pci_mapreg_map(pa, SAFE_BAR, PCI_MAPREG_TYPE_MEM, 0,
  174             &sc->sc_st, &sc->sc_sh, NULL, &iosize, 0)) {
printf(": can't map register space\n");
  176                 goto bad;
  177         }
  179         if (pci_intr_map(pa, &ih)) {
printf(": couldn't map interrupt\n");
  181                 goto bad1;
  182         }
intrstr = pci_intr_string(pa->pa_pc, ih);
sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET, safe_intr, sc,
self->dv_xname);
  186         if (sc->sc_ih == NULL) {
printf(": couldn't establish interrupt");
  188                 if (intrstr != NULL)
printf(" at %s", intrstr);
printf("\n");
  191                 goto bad2;
  192         }
sc->sc_cid = crypto_get_driverid(0);
  195         if (sc->sc_cid < 0) {
printf(": could not get crypto driver id\n");
  197                 goto bad3;
  198         }
sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
  201                 (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
  203         /*
  204          * Allocate packet engine descriptors.
  205          */
  206         if (safe_dma_malloc(sc,
SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
  208             &sc->sc_ringalloc, 0)) {
printf(": cannot allocate PE descriptor ring\n");
  210                 goto bad4;
  211         }
  212         /*
  213          * Hookup the static portion of all our data structures.
  214          */
sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
sc->sc_front = sc->sc_ring;
sc->sc_back = sc->sc_ring;
raddr = sc->sc_ringalloc.dma_paddr;
bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
  221         for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
  222                 struct safe_ringentry *re = &sc->sc_ring[i];
re->re_desc.d_sa = raddr +
offsetof(struct safe_ringentry, re_sa);
re->re_sa.sa_staterec = raddr +
offsetof(struct safe_ringentry, re_sastate);
raddr += sizeof (struct safe_ringentry);
  230         }
  232         /*
  233          * Allocate scatter and gather particle descriptors.
  234          */
  235         if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
  236             &sc->sc_spalloc, 0)) {
printf(": cannot allocate source particle descriptor ring\n");
safe_dma_free(sc, &sc->sc_ringalloc);
  239                 goto bad4;
  240         }
sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
sc->sc_spfree = sc->sc_spring;
bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
  246         if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
  247             &sc->sc_dpalloc, 0)) {
printf(": cannot allocate destination particle "
  249                         "descriptor ring\n");
safe_dma_free(sc, &sc->sc_spalloc);
safe_dma_free(sc, &sc->sc_ringalloc);
  252                 goto bad4;
  253         }
sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
sc->sc_dpfree = sc->sc_dpring;
bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
printf(":");
devinfo = READ_REG(sc, SAFE_DEVINFO);
  262         if (devinfo & SAFE_DEVINFO_RNG)
printf(" RNG");
bzero(algs, sizeof(algs));
  266         if (devinfo & SAFE_DEVINFO_PKEY) {
printf(" PK");
algs[CRK_MOD_EXP] = CRYPTO_ALG_FLAG_SUPPORTED;
crypto_kregister(sc->sc_cid, algs, safe_kprocess);
timeout_set(&sc->sc_pkto, safe_kpoll, sc);
  271         }
bzero(algs, sizeof(algs));
  274         if (devinfo & SAFE_DEVINFO_DES) {
printf(" 3DES");
algs[CRYPTO_3DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
algs[CRYPTO_DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
  278         }
  279         if (devinfo & SAFE_DEVINFO_AES) {
printf(" AES");
algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
  282         }
  283         if (devinfo & SAFE_DEVINFO_MD5) {
printf(" MD5");
algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
  286         }
  287         if (devinfo & SAFE_DEVINFO_SHA1) {
printf(" SHA1");
algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
  290         }
crypto_register(sc->sc_cid, algs, safe_newsession,
safe_freesession, safe_process);
  293         /* XXX other supported algorithms? */
printf(", %s\n", intrstr);
safe_reset_board(sc);           /* reset h/w */
safe_init_pciregs(sc);          /* init pci settings */
safe_init_board(sc);            /* init h/w */
  301         if (devinfo & SAFE_DEVINFO_RNG) {
safe_rng_init(sc);
timeout_set(&sc->sc_rngto, safe_rng, sc);
timeout_add(&sc->sc_rngto, hz * safe_rnginterval);
  306         }
  307         return;
bad4:
  310         /* XXX crypto_unregister_all(sc->sc_cid); */
bad3:
pci_intr_disestablish(pa->pa_pc, sc->sc_ih);
bad2:
  314         /* pci_intr_unmap? */;
bad1:
bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
bad:
  318         return;
  319 }
  321 int
safe_process(struct cryptop *crp)
  323 {
  324         int err = 0, i, nicealign, uniform, s;
  325         struct safe_softc *sc;
  326         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
  327         int bypass, oplen, ivsize, card;
int16_t coffset;
  329         struct safe_session *ses;
  330         struct safe_ringentry *re;
  331         struct safe_sarec *sa;
  332         struct safe_pdesc *pd;
u_int32_t cmd0, cmd1, staterec, iv[4];
s = splnet();
  336         if (crp == NULL || crp->crp_callback == NULL) {
safestats.st_invalid++;
splx(s);
  339                 return (EINVAL);
  340         }
card = SAFE_CARD(crp->crp_sid);
  342         if (card >= safe_cd.cd_ndevs || safe_cd.cd_devs[card] == NULL) {
safestats.st_invalid++;
splx(s);
  345                 return (EINVAL);
  346         }
sc = safe_cd.cd_devs[card];
  349         if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
safestats.st_badsession++;
splx(s);
  352                 return (EINVAL);
  353         }
  355         if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
safestats.st_ringfull++;
splx(s);
  358                 return (ERESTART);
  359         }
re = sc->sc_front;
staterec = re->re_sa.sa_staterec;       /* save */
  363         /* NB: zero everything but the PE descriptor */
bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
re->re_sa.sa_staterec = staterec;       /* restore */
re->re_crp = crp;
re->re_sesn = SAFE_SESSION(crp->crp_sid);
  370         if (crp->crp_flags & CRYPTO_F_IMBUF) {
re->re_src_m = (struct mbuf *)crp->crp_buf;
re->re_dst_m = (struct mbuf *)crp->crp_buf;
  373         } else if (crp->crp_flags & CRYPTO_F_IOV) {
re->re_src_io = (struct uio *)crp->crp_buf;
re->re_dst_io = (struct uio *)crp->crp_buf;
  376         } else {
safestats.st_badflags++;
err = EINVAL;
  379                 goto errout;    /* XXX we don't handle contiguous blocks! */
  380         }
sa = &re->re_sa;
ses = &sc->sc_sessions[re->re_sesn];
crd1 = crp->crp_desc;
  386         if (crd1 == NULL) {
safestats.st_nodesc++;
err = EINVAL;
  389                 goto errout;
  390         }
crd2 = crd1->crd_next;
cmd0 = SAFE_SA_CMD0_BASIC;              /* basic group operation */
cmd1 = 0;
  395         if (crd2 == NULL) {
  396                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC) {
maccrd = crd1;
enccrd = NULL;
cmd0 |= SAFE_SA_CMD0_OP_HASH;
  401                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC) {
maccrd = NULL;
enccrd = crd1;
cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
  407                 } else {
safestats.st_badalg++;
err = EINVAL;
  410                         goto errout;
  411                 }
  412         } else {
  413                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
  415                    (crd2->crd_alg == CRYPTO_DES_CBC ||
crd2->crd_alg == CRYPTO_3DES_CBC ||
crd2->crd_alg == CRYPTO_AES_CBC) &&
  418                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
maccrd = crd1;
enccrd = crd2;
  421                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
crd1->crd_alg == CRYPTO_3DES_CBC ||
crd1->crd_alg == CRYPTO_AES_CBC) &&
  424                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
  426                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
enccrd = crd1;
maccrd = crd2;
  429                 } else {
safestats.st_badalg++;
err = EINVAL;
  432                         goto errout;
  433                 }
cmd0 |= SAFE_SA_CMD0_OP_BOTH;
  435         }
  437         if (enccrd) {
  438                 if (enccrd->crd_alg == CRYPTO_DES_CBC) {
cmd0 |= SAFE_SA_CMD0_DES;
cmd1 |= SAFE_SA_CMD1_CBC;
ivsize = 2*sizeof(u_int32_t);
  442                 } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
cmd0 |= SAFE_SA_CMD0_3DES;
cmd1 |= SAFE_SA_CMD1_CBC;
ivsize = 2*sizeof(u_int32_t);
  446                 } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
cmd0 |= SAFE_SA_CMD0_AES;
cmd1 |= SAFE_SA_CMD1_CBC;
  449                         if (ses->ses_klen == 128)
cmd1 |=  SAFE_SA_CMD1_AES128;
  451                         else if (ses->ses_klen == 192)
cmd1 |=  SAFE_SA_CMD1_AES192;
  453                         else
cmd1 |=  SAFE_SA_CMD1_AES256;
ivsize = 4*sizeof(u_int32_t);
  456                 } else {
cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
ivsize = 0;
  459                 }
  461                 /*
  462                  * Setup encrypt/decrypt state.  When using basic ops
  463                  * we can't use an inline IV because hash/crypt offset
  464                  * must be from the end of the IV to the start of the
  465                  * crypt data and this leaves out the preceding header
  466                  * from the hash calculation.  Instead we place the IV
  467                  * in the state record and set the hash/crypt offset to
  468                  * copy both the header+IV.
  469                  */
  470                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
cmd0 |= SAFE_SA_CMD0_OUTBOUND;
  473                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, iv, ivsize);
  475                         else
bcopy(ses->ses_iv, iv, ivsize);
  477                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
  478                                 if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copyback(re->re_src_m,
enccrd->crd_inject, ivsize, iv);
  481                                 else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copyback(re->re_src_io,
enccrd->crd_inject, ivsize, iv);
  484                         }
  485                         for (i = 0; i < ivsize / sizeof(iv[0]); i++)
re->re_sastate.sa_saved_iv[i] = htole32(iv[i]);
cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
  489                 } else {
cmd0 |= SAFE_SA_CMD0_INBOUND;
  492                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
bcopy(enccrd->crd_iv, iv, ivsize);
  494                         else if (crp->crp_flags & CRYPTO_F_IMBUF)
m_copydata(re->re_src_m, enccrd->crd_inject,
ivsize, (caddr_t)iv);
  497                         else if (crp->crp_flags & CRYPTO_F_IOV)
cuio_copydata(re->re_src_io, enccrd->crd_inject,
ivsize, (caddr_t)iv);
  500                         for (i = 0; i < ivsize / sizeof(iv[0]); i++)
re->re_sastate.sa_saved_iv[i] = htole32(iv[i]);
cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
  503                 }
  504                 /*
  505                  * For basic encryption use the zero pad algorithm.
  506                  * This pads results to an 8-byte boundary and
  507                  * suppresses padding verification for inbound (i.e.
  508                  * decrypt) operations.
  509                  *
  510                  * NB: Not sure if the 8-byte pad boundary is a problem.
  511                  */
cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
  514                 /* XXX assert key bufs have the same size */
  515                 for (i = 0; i < sizeof(sa->sa_key)/sizeof(sa->sa_key[0]); i++)
sa->sa_key[i] = ses->ses_key[i];
  517         }
  519         if (maccrd) {
  520                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
cmd0 |= SAFE_SA_CMD0_MD5;
cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
  523                 } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
cmd0 |= SAFE_SA_CMD0_SHA1;
cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
  526                 } else {
cmd0 |= SAFE_SA_CMD0_HASH_NULL;
  528                 }
  529                 /*
  530                  * Digest data is loaded from the SA and the hash
  531                  * result is saved to the state block where we
  532                  * retrieve it for return to the caller.
  533                  */
  534                 /* XXX assert digest bufs have the same size */
  535                 for (i = 0;
i < sizeof(sa->sa_outdigest)/sizeof(sa->sa_outdigest[i]);
i++) {
sa->sa_indigest[i] = ses->ses_hminner[i];
sa->sa_outdigest[i] = ses->ses_hmouter[i];
  540                 }
cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
  544         }
  546         if (enccrd && maccrd) {
  547                 /*
  548                  * The offset from hash data to the start of
  549                  * crypt data is the difference in the skips.
  550                  */
bypass = maccrd->crd_skip;
coffset = enccrd->crd_skip - maccrd->crd_skip;
  553                 if (coffset < 0) {
DPRINTF(("%s: hash does not precede crypt; "
  555                                 "mac skip %u enc skip %u\n",
__func__, maccrd->crd_skip, enccrd->crd_skip));
safestats.st_skipmismatch++;
err = EINVAL;
  559                         goto errout;
  560                 }
oplen = enccrd->crd_skip + enccrd->crd_len;
  562                 if (maccrd->crd_skip + maccrd->crd_len != oplen) {
DPRINTF(("%s: hash amount %u != crypt amount %u\n",
__func__, maccrd->crd_skip + maccrd->crd_len,
oplen));
safestats.st_lenmismatch++;
err = EINVAL;
  568                         goto errout;
  569                 }
  570 #ifdef SAFE_DEBUG
  571                 if (safe_debug) {
printf("mac: skip %d, len %d, inject %d\n",
maccrd->crd_skip, maccrd->crd_len,
maccrd->crd_inject);
printf("enc: skip %d, len %d, inject %d\n",
enccrd->crd_skip, enccrd->crd_len,
enccrd->crd_inject);
printf("bypass %d coffset %d oplen %d\n",
bypass, coffset, oplen);
  580                 }
  581 #endif
  582                 if (coffset & 3) {      /* offset must be 32-bit aligned */
DPRINTF(("%s: coffset %u misaligned\n",
__func__, coffset));
safestats.st_coffmisaligned++;
err = EINVAL;
  587                         goto errout;
  588                 }
coffset >>= 2;
  590                 if (coffset > 255) {    /* offset must be <256 dwords */
DPRINTF(("%s: coffset %u too big\n",
__func__, coffset));
safestats.st_cofftoobig++;
err = EINVAL;
  595                         goto errout;
  596                 }
  597                 /*
  598                  * Tell the hardware to copy the header to the output.
  599                  * The header is defined as the data from the end of
  600                  * the bypass to the start of data to be encrypted. 
  601                  * Typically this is the inline IV.  Note that you need
  602                  * to do this even if src+dst are the same; it appears
  603                  * that w/o this bit the crypted data is written
  604                  * immediately after the bypass data.
  605                  */
cmd1 |= SAFE_SA_CMD1_HDRCOPY;
  607                 /*
  608                  * Disable IP header mutable bit handling.  This is
  609                  * needed to get correct HMAC calculations.
  610                  */
cmd1 |= SAFE_SA_CMD1_MUTABLE;
  612         } else {
  613                 if (enccrd) {
bypass = enccrd->crd_skip;
oplen = bypass + enccrd->crd_len;
  616                 } else {
bypass = maccrd->crd_skip;
oplen = bypass + maccrd->crd_len;
  619                 }
coffset = 0;
  621         }
  622         /* XXX verify multiple of 4 when using s/g */
  623         if (bypass > 96) {              /* bypass offset must be <= 96 bytes */
DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
safestats.st_bypasstoobig++;
err = EINVAL;
  627                 goto errout;
  628         }
  630         if (bus_dmamap_create(sc->sc_dmat, SAFE_MAX_DMA, SAFE_MAX_PART,
SAFE_MAX_DSIZE, SAFE_MAX_DSIZE, BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
  632             &re->re_src_map)) {
safestats.st_nomap++;
err = ENOMEM;
  635                 goto errout;
  636         }
  637         if (crp->crp_flags & CRYPTO_F_IMBUF) {
  638                 if (bus_dmamap_load_mbuf(sc->sc_dmat, re->re_src_map,
re->re_src_m, BUS_DMA_NOWAIT)) {
bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
re->re_src_map = NULL;
safestats.st_noload++;
err = ENOMEM;
  644                         goto errout;
  645                 }
  646         } else if (crp->crp_flags & CRYPTO_F_IOV) {
  647                 if (bus_dmamap_load_uio(sc->sc_dmat, re->re_src_map,
re->re_src_io, BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
re->re_src_map = NULL;
safestats.st_noload++;
err = ENOMEM;
  653                         goto errout;
  654                 }
  655         }
nicealign = safe_dmamap_aligned(&re->re_src);
uniform = safe_dmamap_uniform(&re->re_src);
DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
nicealign, uniform, re->re_src_nsegs));
  661         if (re->re_src_nsegs > 1) {
re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
  663                         ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
  664                 for (i = 0; i < re->re_src_nsegs; i++) {
  665                         /* NB: no need to check if there's space */
pd = sc->sc_spfree;
  667                         if (++(sc->sc_spfree) == sc->sc_springtop)
sc->sc_spfree = sc->sc_spring;
KASSERT_X((pd->pd_flags&3) == 0 ||
  671                                 (pd->pd_flags&3) == SAFE_PD_DONE,
  672                                 ("bogus source particle descriptor; flags %x",
pd->pd_flags));
pd->pd_addr = re->re_src_segs[i].ds_addr;
pd->pd_ctrl = SAFE_PD_READY |
  676                             ((re->re_src_segs[i].ds_len << SAFE_PD_LEN_S)
  677                             & SAFE_PD_LEN_M);
  678                 }
cmd0 |= SAFE_SA_CMD0_IGATHER;
  680         } else {
  681                 /*
  682                  * No need for gather, reference the operand directly.
  683                  */
re->re_desc.d_src = re->re_src_segs[0].ds_addr;
  685         }
  687         if (enccrd == NULL && maccrd != NULL) {
  688                 /*
  689                  * Hash op; no destination needed.
  690                  */
  691         } else {
  692                 if (crp->crp_flags & CRYPTO_F_IOV) {
  693                         if (!nicealign) {
safestats.st_iovmisaligned++;
err = EINVAL;
  696                                 goto errout;
  697                         }
  698                         if (uniform != 1) {
  699                                 /*
  700                                  * Source is not suitable for direct use as
  701                                  * the destination.  Create a new scatter/gather
  702                                  * list based on the destination requirements
  703                                  * and check if that's ok.
  704                                  */
  705                                 if (bus_dmamap_create(sc->sc_dmat,
SAFE_MAX_DMA, SAFE_MAX_PART,
SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
  709                                     &re->re_dst_map)) {
safestats.st_nomap++;
err = ENOMEM;
  712                                         goto errout;
  713                                 }
  714                                 if (bus_dmamap_load_uio(sc->sc_dmat,
re->re_dst_map, re->re_dst_io,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat,
re->re_dst_map);
re->re_dst_map = NULL;
safestats.st_noload++;
err = ENOMEM;
  722                                         goto errout;
  723                                 }
uniform = safe_dmamap_uniform(&re->re_dst);
  725                                 if (!uniform) {
  726                                         /*
  727                                          * There's no way to handle the DMA
  728                                          * requirements with this uio.  We
  729                                          * could create a separate DMA area for
  730                                          * the result and then copy it back,
  731                                          * but for now we just bail and return
  732                                          * an error.  Note that uio requests
  733                                          * > SAFE_MAX_DSIZE are handled because
  734                                          * the DMA map and segment list for the
  735                                          * destination will result in a
  736                                          * destination particle list that does
  737                                          * the necessary scatter DMA.
  738                                          */ 
safestats.st_iovnotuniform++;
err = EINVAL;
  741                                         goto errout;
  742                                 }
  743                         } else
re->re_dst = re->re_src;
  745                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
  746                         if (nicealign && uniform == 1) {
  747                                 /*
  748                                  * Source layout is suitable for direct
  749                                  * sharing of the DMA map and segment list.
  750                                  */
re->re_dst = re->re_src;
  752                         } else if (nicealign && uniform == 2) {
  753                                 /*
  754                                  * The source is properly aligned but requires a
  755                                  * different particle list to handle DMA of the
  756                                  * result.  Create a new map and do the load to
  757                                  * create the segment list.  The particle
  758                                  * descriptor setup code below will handle the
  759                                  * rest.
  760                                  */
  761                                 if (bus_dmamap_create(sc->sc_dmat,
SAFE_MAX_DMA, SAFE_MAX_PART,
SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
  765                                     &re->re_dst_map)) {
safestats.st_nomap++;
err = ENOMEM;
  768                                         goto errout;
  769                                 }
  770                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
re->re_dst_map, re->re_dst_m,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat,
re->re_dst_map);
re->re_dst_map = NULL;
safestats.st_noload++;
err = ENOMEM;
  778                                         goto errout;
  779                                 }
  780                         } else {                /* !(aligned and/or uniform) */
  781                                 int totlen, len;
  782                                 struct mbuf *m, *top, **mp;
  784                                 /*
  785                                  * DMA constraints require that we allocate a
  786                                  * new mbuf chain for the destination.  We
  787                                  * allocate an entire new set of mbufs of
  788                                  * optimal/required size and then tell the
  789                                  * hardware to copy any bits that are not
  790                                  * created as a byproduct of the operation.
  791                                  */
  792                                 if (!nicealign)
safestats.st_unaligned++;
  794                                 if (!uniform)
safestats.st_notuniform++;
totlen = re->re_src_mapsize;
  797                                 if (re->re_src_m->m_flags & M_PKTHDR) {
len = MHLEN;
MGETHDR(m, M_DONTWAIT, MT_DATA);
  800                                 } else {
len = MLEN;
MGET(m, M_DONTWAIT, MT_DATA);
  803                                 }
  804                                 if (m == NULL) {
safestats.st_nombuf++;
err = sc->sc_nqchip ? ERESTART : ENOMEM;
  807                                         goto errout;
  808                                 }
  809                                 if (len == MHLEN)
M_DUP_PKTHDR(m, re->re_src_m);
  811                                 if (totlen >= MINCLSIZE) {
MCLGET(m, M_DONTWAIT);
  813                                         if ((m->m_flags & M_EXT) == 0) {
m_free(m);
safestats.st_nomcl++;
err = sc->sc_nqchip ?
ERESTART : ENOMEM;
  818                                                 goto errout;
  819                                         }
len = MCLBYTES;
  821                                 }
m->m_len = len;
top = NULL;
mp = &top;
  826                                 while (totlen > 0) {
  827                                         if (top) {
MGET(m, M_DONTWAIT, MT_DATA);
  829                                                 if (m == NULL) {
m_freem(top);
safestats.st_nombuf++;
err = sc->sc_nqchip ?
ERESTART : ENOMEM;
  834                                                         goto errout;
  835                                                 }
len = MLEN;
  837                                         }
  838                                         if (top && totlen >= MINCLSIZE) {
MCLGET(m, M_DONTWAIT);
  840                                                 if ((m->m_flags & M_EXT) == 0) {
  841                                                         *mp = m;
m_freem(top);
safestats.st_nomcl++;
err = sc->sc_nqchip ?
ERESTART : ENOMEM;
  846                                                         goto errout;
  847                                                 }
len = MCLBYTES;
  849                                         }
m->m_len = len = min(totlen, len);
totlen -= len;
  852                                         *mp = m;
mp = &m->m_next;
  854                                 }
re->re_dst_m = top;
  856                                 if (bus_dmamap_create(sc->sc_dmat, 
SAFE_MAX_DMA, SAFE_MAX_PART,
SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
  860                                     &re->re_dst_map) != 0) {
safestats.st_nomap++;
err = ENOMEM;
  863                                         goto errout;
  864                                 }
  865                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
re->re_dst_map, re->re_dst_m,
BUS_DMA_NOWAIT) != 0) {
bus_dmamap_destroy(sc->sc_dmat,
re->re_dst_map);
re->re_dst_map = NULL;
safestats.st_noload++;
err = ENOMEM;
  873                                         goto errout;
  874                                 }
  875                                 if (re->re_src_mapsize > oplen) {
  876                                         /*
  877                                          * There's data following what the
  878                                          * hardware will copy for us.  If this
  879                                          * isn't just the ICV (that's going to
  880                                          * be written on completion), copy it
  881                                          * to the new mbufs
  882                                          */
  883                                         if (!(maccrd &&
  884                                             (re->re_src_mapsize-oplen) == 12 &&
maccrd->crd_inject == oplen))
safe_mcopy(re->re_src_m,
re->re_dst_m,
oplen);
  889                                         else
safestats.st_noicvcopy++;
  891                                 }
  892                         }
  893                 } else {
safestats.st_badflags++;
err = EINVAL;
  896                         goto errout;
  897                 }
  899                 if (re->re_dst_nsegs > 1) {
re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
  901                             ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
  902                         for (i = 0; i < re->re_dst_nsegs; i++) {
pd = sc->sc_dpfree;
KASSERT_X((pd->pd_flags&3) == 0 ||
  905                                         (pd->pd_flags&3) == SAFE_PD_DONE,
  906                                         ("bogus dest particle descriptor; flags %x",
pd->pd_flags));
  908                                 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
sc->sc_dpfree = sc->sc_dpring;
pd->pd_addr = re->re_dst_segs[i].ds_addr;
pd->pd_ctrl = SAFE_PD_READY;
  912                         }
cmd0 |= SAFE_SA_CMD0_OSCATTER;
  914                 } else {
  915                         /*
  916                          * No need for scatter, reference the operand directly.
  917                          */
re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
  919                 }
  920         }
  922         /*
  923          * All done with setup; fillin the SA command words
  924          * and the packet engine descriptor.  The operation
  925          * is now ready for submission to the hardware.
  926          */
sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
sa->sa_cmd1 = cmd1
  929             | (coffset << SAFE_SA_CMD1_OFFSET_S)
  930             | SAFE_SA_CMD1_SAREV1       /* Rev 1 SA data structure */
  931             | SAFE_SA_CMD1_SRPCI;
  933         /*
  934          * NB: the order of writes is important here.  In case the
  935          * chip is scanning the ring because of an outstanding request
  936          * it might nab this one too.  In that case we need to make
  937          * sure the setup is complete before we write the length
  938          * field of the descriptor as it signals the descriptor is
  939          * ready for processing.
  940          */
re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
  942         if (maccrd)
re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
re->re_desc.d_len = oplen
  945                           | SAFE_PE_LEN_READY
  946                           | (bypass << SAFE_PE_LEN_BYPASS_S)
  947                           ;
safestats.st_ipackets++;
safestats.st_ibytes += oplen;
  952         if (++(sc->sc_front) == sc->sc_ringtop)
sc->sc_front = sc->sc_ring;
  955         /* XXX honor batching */
safe_feed(sc, re);
splx(s);
  958         return (0);
errout:
  961         if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
m_freem(re->re_dst_m);
  964         if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
bus_dmamap_unload(sc->sc_dmat, re->re_dst_map);
bus_dmamap_destroy(sc->sc_dmat, re->re_dst_map);
  967         }
  968         if (re->re_src_map != NULL) {
bus_dmamap_unload(sc->sc_dmat, re->re_src_map);
bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
  971         }
crp->crp_etype = err;
crypto_done(crp);
splx(s);
  975         return (err);
  976 }
  978 /*
  979  * Resets the board.  Values in the regesters are left as is
  980  * from the reset (i.e. initial values are assigned elsewhere).
  981  */
  982 void
safe_reset_board(struct safe_softc *sc)
  984 {
u_int32_t v;
  987         /*
  988          * Reset the device.  The manual says no delay
  989          * is needed between marking and clearing reset.
  990          */
v = READ_REG(sc, SAFE_PE_DMACFG) &
  992             ~(SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
SAFE_PE_DMACFG_SGRESET);
WRITE_REG(sc, SAFE_PE_DMACFG, v
  995             | SAFE_PE_DMACFG_PERESET
  996             | SAFE_PE_DMACFG_PDRRESET
  997             | SAFE_PE_DMACFG_SGRESET);
WRITE_REG(sc, SAFE_PE_DMACFG, v);
  999 }
 1001 /*
 1002  * Initialize registers we need to touch only once.
 1003  */
 1004 void
safe_init_board(struct safe_softc *sc)
 1006 {
u_int32_t v, dwords;
v = READ_REG(sc, SAFE_PE_DMACFG);
v &= ~(SAFE_PE_DMACFG_PEMODE | SAFE_PE_DMACFG_ESPACKET);
v |= SAFE_PE_DMACFG_FSENA               /* failsafe enable */
 1012           |  SAFE_PE_DMACFG_GPRPCI              /* gather ring on PCI */
 1013           |  SAFE_PE_DMACFG_SPRPCI              /* scatter ring on PCI */
 1014           |  SAFE_PE_DMACFG_ESDESC              /* endian-swap descriptors */
 1015           |  SAFE_PE_DMACFG_ESPDESC             /* endian-swap part. desc's */
 1016           |  SAFE_PE_DMACFG_ESSA                /* endian-swap SA data */
 1017           ;
WRITE_REG(sc, SAFE_PE_DMACFG, v);
WRITE_REG(sc, SAFE_CRYPTO_CTRL, SAFE_CRYPTO_CTRL_PKEY |
SAFE_CRYPTO_CTRL_3DES | SAFE_CRYPTO_CTRL_RNG);
 1023 #if BYTE_ORDER == LITTLE_ENDIAN
WRITE_REG(sc, SAFE_ENDIAN, SAFE_ENDIAN_TGT_PASS|SAFE_ENDIAN_DMA_PASS);
 1025 #elif BYTE_ORDER == BIG_ENDIAN
WRITE_REG(sc, SAFE_ENDIAN, SAFE_ENDIAN_TGT_PASS|SAFE_ENDIAN_DMA_SWAB);
 1027 #endif
 1029         if (sc->sc_chiprev == SAFE_REV(1,0)) {
 1030                 /*
 1031                  * Avoid large PCI DMA transfers.  Rev 1.0 has a bug where
 1032                  * "target mode transfers" done while the chip is DMA'ing
 1033                  * >1020 bytes cause the hardware to lockup.  To avoid this
 1034                  * we reduce the max PCI transfer size and use small source
 1035                  * particle descriptors (<= 256 bytes).
 1036                  */
WRITE_REG(sc, SAFE_DMA_CFG, 256);
printf("%s: Reduce max DMA size to %u words for rev %u.%u WAR\n",
sc->sc_dev.dv_xname,
 1040                     (READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
SAFE_REV_MAJ(sc->sc_chiprev),
SAFE_REV_MIN(sc->sc_chiprev));
 1043         }
 1045         /* NB: operands+results are overlaid */
WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
 1048         /*
 1049          * Configure ring entry size and number of items in the ring.
 1050          */
KASSERT_X((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
 1052             ("PE ring entry not 32-bit aligned!"));
dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
WRITE_REG(sc, SAFE_PE_RINGCFG,
 1055             (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
WRITE_REG(sc, SAFE_PE_RINGPOLL, 0);     /* disable polling */
WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
WRITE_REG(sc, SAFE_PE_PARTSIZE,
 1061             (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
 1062         /*
 1063          * NB: destination particles are fixed size.  We use
 1064          *     an mbuf cluster and require all results go to
 1065          *     clusters or smaller.
 1066          */
WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
WRITE_REG(sc, SAFE_HI_CLR, SAFE_INT_PE_CDONE | SAFE_INT_PE_DDONE |
SAFE_INT_PE_ERROR | SAFE_INT_PE_ODONE);
 1072         /* it's now safe to enable PE mode, do it */
WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
 1075         /*
 1076          * Configure hardware to use level-triggered interrupts and
 1077          * to interrupt after each descriptor is processed.
 1078          */
DELAY(1000);
WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
DELAY(1000);
WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
DELAY(1000);
WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
DELAY(1000);
 1086 }
 1088 /*
 1089  * Init PCI registers
 1090  */
 1091 void
safe_init_pciregs(struct safe_softc *sc)
 1093 {
 1094 }
 1096 int
safe_dma_malloc(struct safe_softc *sc, bus_size_t size,
 1098     struct safe_dma_alloc *dma, int mapflags)
 1099 {
 1100         int r;
 1102         if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
 1103             &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
 1104                 goto fail_0;
 1106         if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
 1108                 goto fail_1;
 1110         if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
 1112                 goto fail_2;
 1114         if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
size, NULL, BUS_DMA_NOWAIT)) != 0)
 1116                 goto fail_3;
dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
dma->dma_size = size;
 1120         return (0);
fail_3:
bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
fail_2:
bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
fail_1:
bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
fail_0:
dma->dma_map = NULL;
 1130         return (r);
 1131 }
 1133 void
safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
 1135 {
bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
 1140 }
 1143 #define SAFE_RNG_MAXWAIT        1000
 1145 void
safe_rng_init(struct safe_softc *sc)
 1147 {
u_int32_t w, v;
 1149         int i;
WRITE_REG(sc, SAFE_RNG_CTRL, 0);
 1152         /* use default value according to the manual */
WRITE_REG(sc, SAFE_RNG_CNFG, 0x834);    /* magic from SafeNet */
WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 1156         /*
 1157          * There is a bug in rev 1.0 of the 1140 that when the RNG
 1158          * is brought out of reset the ready status flag does not
 1159          * work until the RNG has finished its internal initialization.
 1160          *
 1161          * So in order to determine the device is through its
 1162          * initialization we must read the data register, using the
 1163          * status reg in the read in case it is initialized.  Then read
 1164          * the data register until it changes from the first read.
 1165          * Once it changes read the data register until it changes
 1166          * again.  At this time the RNG is considered initialized. 
 1167          * This could take between 750ms - 1000ms in time.
 1168          */
i = 0;
w = READ_REG(sc, SAFE_RNG_OUT);
 1171         do {
v = READ_REG(sc, SAFE_RNG_OUT);
 1173                 if (v != w) {
w = v;
 1175                         break;
 1176                 }
DELAY(10);
 1178         } while (++i < SAFE_RNG_MAXWAIT);
 1180         /* Wait Until data changes again */
i = 0;
 1182         do {
v = READ_REG(sc, SAFE_RNG_OUT);
 1184                 if (v != w)
 1185                         break;
DELAY(10);
 1187         } while (++i < SAFE_RNG_MAXWAIT);
 1188 }
 1190 __inline u_int32_t
safe_rng_read(struct safe_softc *sc)
 1192 {
 1193         int i;
i = 0;
 1196         while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
 1197                 ;
 1198         return (READ_REG(sc, SAFE_RNG_OUT));
 1199 }
 1201 void
safe_rng(void *arg)
 1203 {
 1204         struct safe_softc *sc = arg;
u_int32_t buf[SAFE_RNG_MAXBUFSIZ];      /* NB: maybe move to softc */
u_int maxwords;
 1207         int i;
safestats.st_rng++;
 1210         /*
 1211          * Fetch the next block of data.
 1212          */
maxwords = safe_rngbufsize;
 1214         if (maxwords > SAFE_RNG_MAXBUFSIZ)
maxwords = SAFE_RNG_MAXBUFSIZ;
retry:
 1217         for (i = 0; i < maxwords; i++)
buf[i] = safe_rng_read(sc);
 1219         /*
 1220          * Check the comparator alarm count and reset the h/w if
 1221          * it exceeds our threshold.  This guards against the
 1222          * hardware oscillators resonating with external signals.
 1223          */
 1224         if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
u_int32_t freq_inc, w;
DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
safestats.st_rngalarm++;
WRITE_REG(sc, SAFE_RNG_CTRL, 
READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
freq_inc = 18;
 1233                 for (i = 0; i < 64; i++) {
w = READ_REG(sc, SAFE_RNG_CNFG);
freq_inc = ((w + freq_inc) & 0x3fL);
w = ((w & ~0x3fL) | freq_inc);
WRITE_REG(sc, SAFE_RNG_CNFG, w);
WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 1241                         (void) safe_rng_read(sc);
DELAY(25);
 1244                         if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
WRITE_REG(sc, SAFE_RNG_CTRL,
READ_REG(sc, SAFE_RNG_CTRL) &
 1247                                     ~SAFE_RNG_CTRL_SHORTEN);
 1248                                 goto retry;
 1249                         }
freq_inc = 1;
 1251                 }
WRITE_REG(sc, SAFE_RNG_CTRL,
READ_REG(sc, SAFE_RNG_CTRL) & ~SAFE_RNG_CTRL_SHORTEN);
 1254         } else
WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 1257         for (i = 0; i < maxwords; i++)
add_true_randomness(buf[i]);
timeout_add(&sc->sc_rngto, hz * safe_rnginterval);
 1261 }
 1263 /*
 1264  * Allocate a new 'session' and return an encoded session id.  'sidp'
 1265  * contains our registration id, and should contain an encoded session
 1266  * id on successful allocation.
 1267  */
 1268 int
safe_newsession(u_int32_t *sidp, struct cryptoini *cri)
 1270 {
 1271         struct cryptoini *c, *encini = NULL, *macini = NULL;
 1272         struct safe_softc *sc = NULL;
 1273         struct safe_session *ses = NULL;
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
 1276         int i, sesn;
 1278         if (sidp == NULL || cri == NULL)
 1279                 return (EINVAL);
 1280         for (i = 0; i < safe_cd.cd_ndevs; i++) {
sc = safe_cd.cd_devs[i];
 1282                 if (sc == NULL || sc->sc_cid == (*sidp))
 1283                         break;
 1284         }
 1285         if (sc == NULL)
 1286                 return (EINVAL);
 1288         for (c = cri; c != NULL; c = c->cri_next) {
 1289                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
c->cri_alg == CRYPTO_SHA1_HMAC) {
 1291                         if (macini)
 1292                                 return (EINVAL);
macini = c;
 1294                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
c->cri_alg == CRYPTO_3DES_CBC ||
c->cri_alg == CRYPTO_AES_CBC) {
 1297                         if (encini)
 1298                                 return (EINVAL);
encini = c;
 1300                 } else
 1301                         return (EINVAL);
 1302         }
 1303         if (encini == NULL && macini == NULL)
 1304                 return (EINVAL);
 1305         if (encini) {                   /* validate key length */
 1306                 switch (encini->cri_alg) {
 1307                 case CRYPTO_DES_CBC:
 1308                         if (encini->cri_klen != 64)
 1309                                 return (EINVAL);
 1310                         break;
 1311                 case CRYPTO_3DES_CBC:
 1312                         if (encini->cri_klen != 192)
 1313                                 return (EINVAL);
 1314                         break;
 1315                 case CRYPTO_AES_CBC:
 1316                         if (encini->cri_klen != 128 &&
encini->cri_klen != 192 &&
encini->cri_klen != 256)
 1319                                 return (EINVAL);
 1320                         break;
 1321                 }
 1322         }
 1324         if (sc->sc_sessions == NULL) {
ses = sc->sc_sessions = (struct safe_session *)malloc(
 1326                     sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
 1327                 if (ses == NULL)
 1328                         return (ENOMEM);
sesn = 0;
sc->sc_nsessions = 1;
 1331         } else {
 1332                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 1333                         if (sc->sc_sessions[sesn].ses_used == 0) {
ses = &sc->sc_sessions[sesn];
 1335                                 break;
 1336                         }
 1337                 }
 1339                 if (ses == NULL) {
sesn = sc->sc_nsessions;
ses = (struct safe_session *)malloc((sesn + 1) *
 1342                             sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
 1343                         if (ses == NULL)
 1344                                 return (ENOMEM);
bcopy(sc->sc_sessions, ses, sesn *
 1346                             sizeof(struct safe_session));
bzero(sc->sc_sessions, sesn *
 1348                             sizeof(struct safe_session));
free(sc->sc_sessions, M_DEVBUF);
sc->sc_sessions = ses;
ses = &sc->sc_sessions[sesn];
sc->sc_nsessions++;
 1353                 }
 1354         }
bzero(ses, sizeof(struct safe_session));
ses->ses_used = 1;
 1359         if (encini) {
 1360                 /* get an IV */
get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
ses->ses_klen = encini->cri_klen;
bcopy(encini->cri_key, ses->ses_key, ses->ses_klen / 8);
 1366                 for (i = 0;
i < sizeof(ses->ses_key)/sizeof(ses->ses_key[0]); i++)
ses->ses_key[i] = htole32(ses->ses_key[i]);
 1369         }
 1371         if (macini) {
 1372                 for (i = 0; i < macini->cri_klen / 8; i++)
macini->cri_key[i] ^= HMAC_IPAD_VAL;
 1375                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
MD5Init(&md5ctx);
MD5Update(&md5ctx, macini->cri_key,
macini->cri_klen / 8);
MD5Update(&md5ctx, hmac_ipad_buffer,
HMAC_BLOCK_LEN - (macini->cri_klen / 8));
bcopy(md5ctx.state, ses->ses_hminner,
 1382                             sizeof(md5ctx.state));
 1383                 } else {
SHA1Init(&sha1ctx);
SHA1Update(&sha1ctx, macini->cri_key,
macini->cri_klen / 8);
SHA1Update(&sha1ctx, hmac_ipad_buffer,
HMAC_BLOCK_LEN - (macini->cri_klen / 8));
bcopy(sha1ctx.state, ses->ses_hminner,
 1390                             sizeof(sha1ctx.state));
 1391                 }
 1393                 for (i = 0; i < macini->cri_klen / 8; i++)
macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 1396                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
MD5Init(&md5ctx);
MD5Update(&md5ctx, macini->cri_key,
macini->cri_klen / 8);
MD5Update(&md5ctx, hmac_opad_buffer,
HMAC_BLOCK_LEN - (macini->cri_klen / 8));
bcopy(md5ctx.state, ses->ses_hmouter,
 1403                             sizeof(md5ctx.state));
 1404                 } else {
SHA1Init(&sha1ctx);
SHA1Update(&sha1ctx, macini->cri_key,
macini->cri_klen / 8);
SHA1Update(&sha1ctx, hmac_opad_buffer,
HMAC_BLOCK_LEN - (macini->cri_klen / 8));
bcopy(sha1ctx.state, ses->ses_hmouter,
 1411                             sizeof(sha1ctx.state));
 1412                 }
 1414                 for (i = 0; i < macini->cri_klen / 8; i++)
macini->cri_key[i] ^= HMAC_OPAD_VAL;
 1417                 /* PE is little-endian, insure proper byte order */
 1418                 for (i = 0;
i < sizeof(ses->ses_hminner)/sizeof(ses->ses_hminner[0]);
i++) {
ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
 1423                 }
 1424         }
 1426         *sidp = SAFE_SID(sc->sc_dev.dv_unit, sesn);
 1427         return (0);
 1428 }
 1430 /*
 1431  * Deallocate a session.
 1432  */
 1433 int
safe_freesession(u_int64_t tid)
 1435 {
 1436         struct safe_softc *sc;
 1437         int session, ret, card;
u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
card = SAFE_CARD(sid);
 1441         if (card >= safe_cd.cd_ndevs || safe_cd.cd_devs[card] == NULL)
 1442                 return (EINVAL);
sc = safe_cd.cd_devs[card];
 1445         if (sc == NULL)
 1446                 return (EINVAL);
session = SAFE_SESSION(sid);
 1449         if (session < sc->sc_nsessions) {
bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
ret = 0;
 1452         } else
ret = EINVAL;
 1454         return (ret);
 1455 }
 1457 /*
 1458  * Is the operand suitable aligned for direct DMA.  Each
 1459  * segment must be aligned on a 32-bit boundary and all
 1460  * but the last segment must be a multiple of 4 bytes.
 1461  */
 1462 int
safe_dmamap_aligned(const struct safe_operand *op)
 1464 {
 1465         int i;
 1467         for (i = 0; i < op->map->dm_nsegs; i++) {
 1468                 if (op->map->dm_segs[i].ds_addr & 3)
 1469                         return (0);
 1470                 if (i != (op->map->dm_nsegs - 1) &&
 1471                     (op->map->dm_segs[i].ds_len & 3))
 1472                         return (0);
 1473         }
 1474         return (1);
 1475 }
 1477 /*
 1478  * Clean up after a chip crash.
 1479  * It is assumed that the caller in splnet()
 1480  */
 1481 void
safe_cleanchip(struct safe_softc *sc)
 1483 {
 1485         if (sc->sc_nqchip != 0) {
 1486                 struct safe_ringentry *re = sc->sc_back;
 1488                 while (re != sc->sc_front) {
 1489                         if (re->re_desc.d_csr != 0)
safe_free_entry(sc, re);
 1491                         if (++re == sc->sc_ringtop)
re = sc->sc_ring;
 1493                 }
sc->sc_back = re;
sc->sc_nqchip = 0;
 1496         }
 1497 }
 1499 /*
 1500  * free a safe_q
 1501  * It is assumed that the caller is within splnet().
 1502  */
 1503 int
safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
 1505 {
 1506         struct cryptop *crp;
 1508         /*
 1509          * Free header MCR
 1510          */
 1511         if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
m_freem(re->re_dst_m);
crp = (struct cryptop *)re->re_crp;
re->re_desc.d_csr = 0;
crp->crp_etype = EFAULT;
crypto_done(crp);
 1520         return (0);
 1521 }
 1523 /*
 1524  * safe_feed() - post a request to chip
 1525  */
 1526 void
safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
 1528 {
bus_dmamap_sync(sc->sc_dmat, re->re_src_map,
 1530             0, re->re_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
 1531         if (re->re_dst_map != NULL)
bus_dmamap_sync(sc->sc_dmat, re->re_dst_map, 0,
re->re_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
 1534         /* XXX have no smaller granularity */
safe_dma_sync(sc, &sc->sc_ringalloc, 
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
safe_dma_sync(sc, &sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
safe_dma_sync(sc, &sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
 1540 #ifdef SAFE_DEBUG
 1541         if (safe_debug) {
safe_dump_ringstate(sc, __func__);
safe_dump_request(sc, __func__, re);
 1544         }
 1545 #endif
sc->sc_nqchip++;
 1547         if (sc->sc_nqchip > safestats.st_maxqchip)
safestats.st_maxqchip = sc->sc_nqchip;
 1549         /* poke h/w to check descriptor ring, any value can be written */
WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
 1551 }
 1553 /*
 1554  * Is the operand suitable for direct DMA as the destination
 1555  * of an operation.  The hardware requires that each ``particle''
 1556  * but the last in an operation result have the same size.  We
 1557  * fix that size at SAFE_MAX_DSIZE bytes.  This routine returns
 1558  * 0 if some segment is not a multiple of this size, 1 if all
 1559  * segments are exactly this size, or 2 if segments are at worst
 1560  * a multple of this size.
 1561  */
 1562 int
safe_dmamap_uniform(const struct safe_operand *op)
 1564 {
 1565         int result = 1, i;
 1567         if (op->map->dm_nsegs <= 0)
 1568                 return (result);
 1570         for (i = 0; i < op->map->dm_nsegs-1; i++) {
 1571                 if (op->map->dm_segs[i].ds_len % SAFE_MAX_DSIZE)
 1572                         return (0);
 1573                 if (op->map->dm_segs[i].ds_len != SAFE_MAX_DSIZE)
result = 2;
 1575         }
 1576         return (result);
 1577 }
 1579 /*
 1580  * Copy all data past offset from srcm to dstm.
 1581  */
 1582 void
safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
 1584 {
u_int j, dlen, slen;
caddr_t dptr, sptr;
 1588         /*
 1589          * Advance src and dst to offset.
 1590          */
 1591         for (j = offset; srcm->m_len <= j;) {
j -= srcm->m_len;
srcm = srcm->m_next;
 1594                 if (srcm == NULL)
 1595                         return;
 1596         }
sptr = mtod(srcm, caddr_t) + j;
slen = srcm->m_len - j;
 1600         for (j = offset; dstm->m_len <= j;) {
j -= dstm->m_len;
dstm = dstm->m_next;
 1603                 if (dstm == NULL)
 1604                         return;
 1605         }
dptr = mtod(dstm, caddr_t) + j;
dlen = dstm->m_len - j;
 1609         /*
 1610          * Copy everything that remains.
 1611          */
 1612         for (;;) {
j = min(slen, dlen);
bcopy(sptr, dptr, j);
 1615                 if (slen == j) {
srcm = srcm->m_next;
 1617                         if (srcm == NULL)
 1618                                 return;
sptr = srcm->m_data;
slen = srcm->m_len;
 1621                 } else
sptr += j, slen -= j;
 1623                 if (dlen == j) {
dstm = dstm->m_next;
 1625                         if (dstm == NULL)
 1626                                 return;
dptr = dstm->m_data;
dlen = dstm->m_len;
 1629                 } else
dptr += j, dlen -= j;
 1631         }
 1632 }
 1634 void
safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
 1636 {
 1637         struct cryptop *crp = (struct cryptop *)re->re_crp;
 1638         struct cryptodesc *crd;
safestats.st_opackets++;
safestats.st_obytes += (re->re_dst_map == NULL) ?
re->re_src_mapsize : re->re_dst_mapsize;
safe_dma_sync(sc, &sc->sc_ringalloc,
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 1646         if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
printf("%s: csr 0x%x cmd0 0x%x cmd1 0x%x\n",
sc->sc_dev.dv_xname, re->re_desc.d_csr,
re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
safestats.st_peoperr++;
crp->crp_etype = EIO;           /* something more meaningful? */
 1652         }
 1653         if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
bus_dmamap_sync(sc->sc_dmat, re->re_dst_map, 0,
re->re_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dmat, re->re_dst_map);
bus_dmamap_destroy(sc->sc_dmat, re->re_dst_map);
 1658         }
bus_dmamap_sync(sc->sc_dmat, re->re_src_map, 0,
re->re_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, re->re_src_map);
bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
 1664         /* 
 1665          * If result was written to a different mbuf chain, swap
 1666          * it in as the return value and reclaim the original.
 1667          */
 1668         if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
m_freem(re->re_src_m);
crp->crp_buf = (caddr_t)re->re_dst_m;
 1671         }
 1673         if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
 1674                 /* copy out IV for future use */
 1675                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 1676                         int ivsize;
 1678                         if (crd->crd_alg == CRYPTO_DES_CBC ||
crd->crd_alg == CRYPTO_3DES_CBC) {
ivsize = 2*sizeof(u_int32_t);
 1681                         } else if (crd->crd_alg == CRYPTO_AES_CBC) {
ivsize = 4*sizeof(u_int32_t);
 1683                         } else
 1684                                 continue;
 1685                         if (crp->crp_flags & CRYPTO_F_IMBUF) {
m_copydata((struct mbuf *)crp->crp_buf,
crd->crd_skip + crd->crd_len - ivsize,
ivsize,
 1689                                         (caddr_t) sc->sc_sessions[re->re_sesn].ses_iv);
 1690                         } else if (crp->crp_flags & CRYPTO_F_IOV) {
cuio_copydata((struct uio *)crp->crp_buf,
crd->crd_skip + crd->crd_len - ivsize,
ivsize,
 1694                                         (caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
 1695                         }
 1696                         break;
 1697                 }
 1698         }
 1700         if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
 1701                 /* copy out ICV result */
 1702                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 1703                         if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
crd->crd_alg == CRYPTO_SHA1_HMAC))
 1705                                 continue;
 1706                         if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
 1707                                 /*
 1708                                  * SHA-1 ICV's are byte-swapped; fix 'em up
 1709                                  * before copy them to their destination.
 1710                                  */
bswap32(re->re_sastate.sa_saved_indigest[0]);
bswap32(re->re_sastate.sa_saved_indigest[1]);
bswap32(re->re_sastate.sa_saved_indigest[2]);
 1714                         }
 1715                         if (crp->crp_flags & CRYPTO_F_IMBUF) {
m_copyback((struct mbuf *)crp->crp_buf,
crd->crd_inject, 12,
 1718                                         (caddr_t)re->re_sastate.sa_saved_indigest);
 1719                         } else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) {
bcopy((caddr_t)re->re_sastate.sa_saved_indigest,
crp->crp_mac, 12);
 1722                         }
 1723                         break;
 1724                 }
 1725         }
crypto_done(crp);
 1728 }
 1730 /*
 1731  * SafeXcel Interrupt routine
 1732  */
 1733 int
safe_intr(void *arg)
 1735 {
 1736         struct safe_softc *sc = arg;
 1737         volatile u_int32_t stat;
stat = READ_REG(sc, SAFE_HM_STAT);
 1740         if (stat == 0)                  /* shared irq, not for us */
 1741                 return (0);
WRITE_REG(sc, SAFE_HI_CLR, stat);       /* IACK */
 1745         if ((stat & SAFE_INT_PE_DDONE)) {
 1746                 /*
 1747                  * Descriptor(s) done; scan the ring and
 1748                  * process completed operations.
 1749                  */
 1750                 while (sc->sc_back != sc->sc_front) {
 1751                         struct safe_ringentry *re = sc->sc_back;
 1752 #ifdef SAFE_DEBUG
 1753                         if (safe_debug) {
safe_dump_ringstate(sc, __func__);
safe_dump_request(sc, __func__, re);
 1756                         }
 1757 #endif
 1758                         /*
 1759                          * safe_process marks ring entries that were allocated
 1760                          * but not used with a csr of zero.  This insures the
 1761                          * ring front pointer never needs to be set backwards
 1762                          * in the event that an entry is allocated but not used
 1763                          * because of a setup error.
 1764                          */
 1765                         if (re->re_desc.d_csr != 0) {
 1766                                 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
 1767                                         break;
 1768                                 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
 1769                                         break;
sc->sc_nqchip--;
safe_callback(sc, re);
 1772                         }
 1773                         if (++(sc->sc_back) == sc->sc_ringtop)
sc->sc_back = sc->sc_ring;
 1775                 }
 1776         }
 1778         return (1);
 1779 }
 1781 struct safe_softc *
safe_kfind(struct cryptkop *krp)
 1783 {
 1784         struct safe_softc *sc;
 1785         int i;
 1787         for (i = 0; i < safe_cd.cd_ndevs; i++) {
sc = safe_cd.cd_devs[i];
 1789                 if (sc == NULL)
 1790                         continue;
 1791                 if (sc->sc_cid == krp->krp_hid)
 1792                         return (sc);
 1793         }
 1794         return (NULL);
 1795 }
 1797 int
safe_kprocess(struct cryptkop *krp)
 1799 {
 1800         struct safe_softc *sc;
 1801         struct safe_pkq *q;
 1802         int s;
 1804         if ((sc = safe_kfind(krp)) == NULL) {
krp->krp_status = EINVAL;
 1806                 goto err;
 1807         }
 1809         if (krp->krp_op != CRK_MOD_EXP) {
krp->krp_status = EOPNOTSUPP;
 1811                 goto err;
 1812         }
q = (struct safe_pkq *)malloc(sizeof(*q), M_DEVBUF, M_NOWAIT);
 1815         if (q == NULL) {
krp->krp_status = ENOMEM;
 1817                 goto err;
 1818         }
q->pkq_krp = krp;
s = splnet();
SIMPLEQ_INSERT_TAIL(&sc->sc_pkq, q, pkq_next);
safe_kfeed(sc);
splx(s);
 1825         return (0);
err:
crypto_kdone(krp);
 1829         return (0);
 1830 }
 1832 #define SAFE_CRK_PARAM_BASE     0
 1833 #define SAFE_CRK_PARAM_EXP      1
 1834 #define SAFE_CRK_PARAM_MOD      2
 1836 int
safe_kstart(struct safe_softc *sc)
 1838 {
 1839         struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
 1840         int exp_bits, mod_bits, base_bits;
u_int32_t op, a_off, b_off, c_off, d_off;
 1843         if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
krp->krp_status = EINVAL;
 1845                 return (1);
 1846         }
base_bits = safe_ksigbits(&krp->krp_param[SAFE_CRK_PARAM_BASE]);
 1849         if (base_bits > 2048)
 1850                 goto too_big;
 1851         if (base_bits <= 0)             /* 5. base not zero */
 1852                 goto too_small;
exp_bits = safe_ksigbits(&krp->krp_param[SAFE_CRK_PARAM_EXP]);
 1855         if (exp_bits > 2048)
 1856                 goto too_big;
 1857         if (exp_bits <= 0)              /* 1. exponent word length > 0 */
 1858                 goto too_small;         /* 4. exponent not zero */
mod_bits = safe_ksigbits(&krp->krp_param[SAFE_CRK_PARAM_MOD]);
 1861         if (mod_bits > 2048)
 1862                 goto too_big;
 1863         if (mod_bits <= 32)             /* 2. modulus word length > 1 */
 1864                 goto too_small;         /* 8. MSW of modulus != zero */
 1865         if (mod_bits < exp_bits)        /* 3 modulus len >= exponent len */
 1866                 goto too_small;
 1867         if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
 1868                 goto bad_domain;        /* 6. modulus is odd */
 1869         if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
 1870                 goto too_small;         /* make sure result will fit */
 1872         /* 7. modulus > base */
 1873         if (mod_bits < base_bits)
 1874                 goto too_small;
 1875         if (mod_bits == base_bits) {
u_int8_t *basep, *modp;
 1877                 int i;
basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
 1880                     ((base_bits + 7) / 8) - 1;
modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
 1882                     ((mod_bits + 7) / 8) - 1;
 1884                 for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
 1885                         if (*modp < *basep)
 1886                                 goto too_small;
 1887                         if (*modp > *basep)
 1888                                 break;
 1889                 }
 1890         }
 1892         /* And on the 9th step, he rested. */
WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
 1896         if (mod_bits > 1024) {
op = SAFE_PK_FUNC_EXP4;
a_off = 0x000;
b_off = 0x100;
c_off = 0x200;
d_off = 0x300;
 1902         } else {
op = SAFE_PK_FUNC_EXP16;
a_off = 0x000;
b_off = 0x080;
c_off = 0x100;
d_off = 0x180;
 1908         }
sc->sc_pk_reslen = b_off - a_off;
sc->sc_pk_resoff = d_off;
 1912         /* A is exponent, B is modulus, C is base, D is result */
safe_kload_reg(sc, a_off, b_off - a_off,
 1914             &krp->krp_param[SAFE_CRK_PARAM_EXP]);
WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
safe_kload_reg(sc, b_off, b_off - a_off,
 1917             &krp->krp_param[SAFE_CRK_PARAM_MOD]);
WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
safe_kload_reg(sc, c_off, b_off - a_off,
 1920             &krp->krp_param[SAFE_CRK_PARAM_BASE]);
WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
 1926         return (0);
too_big:
krp->krp_status = E2BIG;
 1930         return (1);
too_small:
krp->krp_status = ERANGE;
 1933         return (1);
bad_domain:
krp->krp_status = EDOM;
 1936         return (1);
 1937 }
 1939 int
safe_ksigbits(struct crparam *cr)
 1941 {
u_int plen = (cr->crp_nbits + 7) / 8;
 1943         int i, sig = plen * 8;
u_int8_t c, *p = cr->crp_p;
 1946         for (i = plen - 1; i >= 0; i--) {
c = p[i];
 1948                 if (c != 0) {
 1949                         while ((c & 0x80) == 0) {
sig--;
c <<= 1;
 1952                         }
 1953                         break;
 1954                 }
sig -= 8;
 1956         }
 1957         return (sig);
 1958 }
 1960 void
safe_kfeed(struct safe_softc *sc)
 1962 {
 1963         if (SIMPLEQ_EMPTY(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
 1964                 return;
 1965         if (sc->sc_pkq_cur != NULL)
 1966                 return;
 1967         while (!SIMPLEQ_EMPTY(&sc->sc_pkq)) {
 1968                 struct safe_pkq *q = SIMPLEQ_FIRST(&sc->sc_pkq);
sc->sc_pkq_cur = q;
SIMPLEQ_REMOVE_HEAD(&sc->sc_pkq, pkq_next);
 1972                 if (safe_kstart(sc) != 0) {
crypto_kdone(q->pkq_krp);
free(q, M_DEVBUF);
sc->sc_pkq_cur = NULL;
 1976                 } else {
 1977                         /* op started, start polling */
timeout_add(&sc->sc_pkto, 1);
 1979                         break;
 1980                 }
 1981         }
 1982 }
 1984 void
safe_kpoll(void *vsc)
 1986 {
 1987         struct safe_softc *sc = vsc;
 1988         struct safe_pkq *q;
 1989         struct crparam *res;
 1990         int s, i;
u_int32_t buf[64];
s = splnet();
 1994         if (sc->sc_pkq_cur == NULL)
 1995                 goto out;
 1996         if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
 1997                 /* still running, check back later */
timeout_add(&sc->sc_pkto, 1);
 1999                 goto out;
 2000         }
q = sc->sc_pkq_cur;
res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
bzero(buf, sizeof(buf));
bzero(res->crp_p, (res->crp_nbits + 7) / 8);
 2006         for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
buf[i] = letoh32(READ_REG(sc, SAFE_PK_RAM_START +
sc->sc_pk_resoff + (i << 2)));
bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
res->crp_nbits = sc->sc_pk_reslen * 8;
res->crp_nbits = safe_ksigbits(res);
 2013         for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
WRITE_REG(sc, i, 0);
crypto_kdone(q->pkq_krp);
free(q, M_DEVBUF);
sc->sc_pkq_cur = NULL;
safe_kfeed(sc);
out:
splx(s);
 2023 }
 2025 void
safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
 2027     struct crparam *n)
 2028 {
u_int32_t buf[64], i;
bzero(buf, sizeof(buf));
bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
 2034         for (i = 0; i < len >> 2; i++)
WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
htole32(buf[i]));
 2037 }
 2039 #ifdef SAFE_DEBUG
 2041 void
safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
 2043 {
printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n", tag,
READ_REG(sc, SAFE_DMA_ENDIAN), READ_REG(sc, SAFE_DMA_SRCADDR),
READ_REG(sc, SAFE_DMA_DSTADDR), READ_REG(sc, SAFE_DMA_STAT));
 2047 }
 2049 void
safe_dump_intrstate(struct safe_softc *sc, const char *tag)
 2051 {
printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n",
tag, READ_REG(sc, SAFE_HI_CFG), READ_REG(sc, SAFE_HI_MASK),
READ_REG(sc, SAFE_HI_DESC_CNT), READ_REG(sc, SAFE_HU_STAT),
READ_REG(sc, SAFE_HM_STAT));
 2056 }
 2058 void
safe_dump_ringstate(struct safe_softc *sc, const char *tag)
 2060 {
u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
 2063         /* NB: assume caller has lock on ring */
printf("%s: ERNGSTAT %x (next %u) back %u front %u\n",
tag, estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
sc->sc_back - sc->sc_ring, sc->sc_front - sc->sc_ring);
 2067 }
 2069 void
safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
 2071 {
 2072         int ix, nsegs;
ix = re - sc->sc_ring;
printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n", tag,
re, ix, re->re_desc.d_csr, re->re_desc.d_src, re->re_desc.d_dst,
re->re_desc.d_sa, re->re_desc.d_len);
 2078         if (re->re_src_nsegs > 1) {
ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
 2080                     sizeof(struct safe_pdesc);
 2081                 for (nsegs = re->re_src_nsegs; nsegs; nsegs--) {
printf(" spd[%u] %p: %p", ix,
 2083                             &sc->sc_spring[ix],
 2084                             (caddr_t)sc->sc_spring[ix].pd_addr);
printf("\n");
 2086                         if (++ix == SAFE_TOTAL_SPART)
ix = 0;
 2088                 }
 2089         }
 2090         if (re->re_dst_nsegs > 1) {
ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
 2092                     sizeof(struct safe_pdesc);
 2093                 for (nsegs = re->re_dst_nsegs; nsegs; nsegs--) {
printf(" dpd[%u] %p: %p\n", ix,
 2095                             &sc->sc_dpring[ix],
 2096                             (caddr_t) sc->sc_dpring[ix].pd_addr);
 2097                         if (++ix == SAFE_TOTAL_DPART)
ix = 0;
 2099                 }
 2100         }
printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
printf("sa: key %x %x %x %x %x %x %x %x\n", re->re_sa.sa_key[0],
re->re_sa.sa_key[1], re->re_sa.sa_key[2], re->re_sa.sa_key[3],
re->re_sa.sa_key[4], re->re_sa.sa_key[5], re->re_sa.sa_key[6],
re->re_sa.sa_key[7]);
printf("sa: indigest %x %x %x %x %x\n", re->re_sa.sa_indigest[0],
re->re_sa.sa_indigest[1], re->re_sa.sa_indigest[2],
re->re_sa.sa_indigest[3], re->re_sa.sa_indigest[4]);
printf("sa: outdigest %x %x %x %x %x\n", re->re_sa.sa_outdigest[0],
re->re_sa.sa_outdigest[1], re->re_sa.sa_outdigest[2],
re->re_sa.sa_outdigest[3], re->re_sa.sa_outdigest[4]);
printf("sr: iv %x %x %x %x\n",
re->re_sastate.sa_saved_iv[0], re->re_sastate.sa_saved_iv[1],
re->re_sastate.sa_saved_iv[2], re->re_sastate.sa_saved_iv[3]);
printf("sr: hashbc %u indigest %x %x %x %x %x\n",
re->re_sastate.sa_saved_hashbc,
re->re_sastate.sa_saved_indigest[0],
re->re_sastate.sa_saved_indigest[1],
re->re_sastate.sa_saved_indigest[2],
re->re_sastate.sa_saved_indigest[3],
re->re_sastate.sa_saved_indigest[4]);
 2123 }
 2125 void
safe_dump_ring(struct safe_softc *sc, const char *tag)
 2127 {
printf("\nSafeNet Ring State:\n");
safe_dump_intrstate(sc, tag);
safe_dump_dmastatus(sc, tag);
safe_dump_ringstate(sc, tag);
 2132         if (sc->sc_nqchip) {
 2133                 struct safe_ringentry *re = sc->sc_back;
 2134                 do {
safe_dump_request(sc, tag, re);
 2136                         if (++re == sc->sc_ringtop)
re = sc->sc_ring;
 2138                 } while (re != sc->sc_front);
 2139         }
 2140 }
 2142 #endif /* SAFE_DEBUG */