root/dev/ic/sili.c

DEFINITIONS

1. sili_attach
2. sili_detach
3. sili_port_intr
4. sili_intr
5. sili_ports_alloc
6. sili_ports_free
7. sili_ccb_alloc
8. sili_ccb_free
9. sili_get_ccb
10. sili_put_ccb
11. sili_dmamem_alloc
12. sili_dmamem_free
13. sili_read
14. sili_write
15. sili_pread
16. sili_pwrite
17. sili_pwait_eq
18. sili_pwait_ne
19. sili_post_direct
20. sili_pread_fis
21. sili_post_indirect
22. sili_signature
23. sili_ata_probe
24. sili_ata_cmd
25. sili_ata_cmd_done
26. sili_ata_cmd_timeout
27. sili_load
28. sili_unload
29. sili_poll
30. sili_start
31. sili_read_ncq_error
32. sili_ata_get_xfer
33. sili_ata_put_xfer

    1 /*      $OpenBSD: sili.c,v 1.33 2007/04/22 00:06:51 dlg Exp $ */
    3 /*
    4  * Copyright (c) 2007 David Gwynne <dlg@openbsd.org>
    5  *
    6  * Permission to use, copy, modify, and distribute this software for any
    7  * purpose with or without fee is hereby granted, provided that the above
    8  * copyright notice and this permission notice appear in all copies.
    9  *
   10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
   12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
   13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
   14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
   15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
   16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
   17  */
   19 #include <sys/param.h>
   20 #include <sys/systm.h>
   21 #include <sys/buf.h>
   22 #include <sys/device.h>
   23 #include <sys/proc.h>
   24 #include <sys/malloc.h>
   25 #include <sys/kernel.h>
   27 #include <machine/bus.h>
   29 #include <dev/ata/atascsi.h>
   31 #include <dev/ic/silireg.h>
   32 #include <dev/ic/silivar.h>
   34 /* use SILI_DEBUG for dmesg spam */
   35 #define NO_SILI_DEBUG
   37 #ifdef SILI_DEBUG
   38 #define SILI_D_VERBOSE          (1<<0)
   39 #define SILI_D_INTR             (1<<1)
   41 int silidebug = SILI_D_VERBOSE;
   43 #define DPRINTF(m, a...)        do { if ((m) & silidebug) printf(a); } while (0)
   44 #else
   45 #define DPRINTF(m, a...)
   46 #endif
   48 struct cfdriver sili_cd = {
NULL, "sili", DV_DULL
   50 };
   52 /* wrapper around dma memory */
   53 struct sili_dmamem {
bus_dmamap_t            sdm_map;
bus_dma_segment_t       sdm_seg;
size_t                  sdm_size;
caddr_t                 sdm_kva;
   58 };
   59 #define SILI_DMA_MAP(_sdm)      ((_sdm)->sdm_map)
   60 #define SILI_DMA_DVA(_sdm)      ((_sdm)->sdm_map->dm_segs[0].ds_addr)
   61 #define SILI_DMA_KVA(_sdm)      ((void *)(_sdm)->sdm_kva)
   63 struct sili_dmamem      *sili_dmamem_alloc(struct sili_softc *, bus_size_t,
bus_size_t);
   65 void                    sili_dmamem_free(struct sili_softc *,
   66                             struct sili_dmamem *);
   68 /* per port goo */
   69 struct sili_ccb;
   71 /* size of scratch space for use in error recovery. */
   72 #define SILI_SCRATCH_LEN        512     /* must be at least 1 sector */
   74 struct sili_port {
   75         struct sili_softc       *sp_sc;
bus_space_handle_t      sp_ioh;
   78         struct sili_ccb         *sp_ccbs;
   79         struct sili_dmamem      *sp_cmds;
   80         struct sili_dmamem      *sp_scratch;
TAILQ_HEAD(, sili_ccb)  sp_free_ccbs;
   84         volatile u_int32_t      sp_active;
TAILQ_HEAD(, sili_ccb)  sp_active_ccbs;
TAILQ_HEAD(, sili_ccb)  sp_deferred_ccbs;
   88 #ifdef SILI_DEBUG
   89         char                    sp_name[16];
   90 #define PORTNAME(_sp)   ((_sp)->sp_name)
   91 #else
   92 #define PORTNAME(_sp)   DEVNAME((_sp)->sp_sc)
   93 #endif
   94 };
   96 int                     sili_ports_alloc(struct sili_softc *);
   97 void                    sili_ports_free(struct sili_softc *);
   99 /* ccb shizz */
  101 /*
  102  * the dma memory for each command will be made up of a prb followed by
  103  * 7 sgts, this is a neat 512 bytes.
  104  */
  105 #define SILI_CMD_LEN            512
  107 /*
  108  * you can fit 22 sge's into 7 sgts and a prb:
  109  * there's 1 sgl in an atapi prb (two in the ata one, but we cant over
  110  * advertise), but that's needed for the chain element. you get three sges
  111  * per sgt cos you lose the 4th sge for the chaining, but you keep it in
  112  * the last sgt. so 3 x 6 + 4 is 22.
  113  */
  114 #define SILI_DMA_SEGS           22
  116 struct sili_ccb {
  117         struct ata_xfer         ccb_xa;
  119         void                    *ccb_cmd;
u_int64_t               ccb_cmd_dva;
bus_dmamap_t            ccb_dmamap;
  123         struct sili_port        *ccb_port;
TAILQ_ENTRY(sili_ccb)   ccb_entry;
  126 };
  128 int                     sili_ccb_alloc(struct sili_port *);
  129 void                    sili_ccb_free(struct sili_port *);
  130 struct sili_ccb         *sili_get_ccb(struct sili_port *);
  131 void                    sili_put_ccb(struct sili_ccb *);
  133 /* bus space ops */
u_int32_t               sili_read(struct sili_softc *, bus_size_t);
  135 void                    sili_write(struct sili_softc *, bus_size_t, u_int32_t);
u_int32_t               sili_pread(struct sili_port *, bus_size_t);
  137 void                    sili_pwrite(struct sili_port *, bus_size_t, u_int32_t);
  138 int                     sili_pwait_eq(struct sili_port *, bus_size_t,
u_int32_t, u_int32_t, int);
  140 int                     sili_pwait_ne(struct sili_port *, bus_size_t,
u_int32_t, u_int32_t, int);
  143 /* command handling */
  144 void                    sili_post_direct(struct sili_port *, u_int,
  145                             void *, size_t buflen);
  146 void                    sili_post_indirect(struct sili_port *,
  147                             struct sili_ccb *);
  148 void                    sili_pread_fis(struct sili_port *, u_int,
  149                             struct ata_fis_d2h *);
u_int32_t               sili_signature(struct sili_port *, u_int);
  151 int                     sili_load(struct sili_ccb *, struct sili_sge *, int);
  152 void                    sili_unload(struct sili_ccb *);
  153 int                     sili_poll(struct sili_ccb *, int, void (*)(void *));
  154 void                    sili_start(struct sili_port *, struct sili_ccb *);
  155 int                     sili_read_ncq_error(struct sili_port *, int *);
  157 /* port interrupt handler */
u_int32_t               sili_port_intr(struct sili_port *, int);
  160 /* atascsi interface */
  161 int                     sili_ata_probe(void *, int);
  162 struct ata_xfer         *sili_ata_get_xfer(void *, int);
  163 void                    sili_ata_put_xfer(struct ata_xfer *);
  164 int                     sili_ata_cmd(struct ata_xfer *);
  166 struct atascsi_methods sili_atascsi_methods = {
sili_ata_probe,
sili_ata_get_xfer,
sili_ata_cmd
  170 };
  172 /* completion paths */
  173 void                    sili_ata_cmd_done(struct sili_ccb *, int);
  174 void                    sili_ata_cmd_timeout(void *);
  176 int
sili_attach(struct sili_softc *sc)
  178 {
  179         struct atascsi_attach_args      aaa;
printf("\n");
  183         if (sili_ports_alloc(sc) != 0) {
  184                 /* error already printed by sili_port_alloc */
  185                 return (1);
  186         }
  188         /* bounce the controller */
sili_write(sc, SILI_REG_GC, SILI_REG_GC_GR);
sili_write(sc, SILI_REG_GC, 0x0);
bzero(&aaa, sizeof(aaa));
aaa.aaa_cookie = sc;
aaa.aaa_methods = &sili_atascsi_methods;
aaa.aaa_minphys = minphys;
aaa.aaa_nports = sc->sc_nports;
aaa.aaa_ncmds = SILI_MAX_CMDS;
aaa.aaa_capability = ASAA_CAP_NCQ;
sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
  202         return (0);
  203 }
  205 int
sili_detach(struct sili_softc *sc, int flags)
  207 {
  208         return (0);
  209 }
u_int32_t
sili_port_intr(struct sili_port *sp, int timeout_slot)
  213 {
u_int32_t                       is, pss_saved, pss_masked;
u_int32_t                       processed = 0, need_restart = 0;
  216         int                             slot;
  217         struct sili_ccb                 *ccb;
is = sili_pread(sp, SILI_PREG_IS);
pss_saved = sili_pread(sp, SILI_PREG_PSS); /* reading acks CMDCOMP */
  222 #ifdef SILI_DEBUG
  223         if ((pss_saved & SILI_PREG_PSS_ALL_SLOTS) != sp->sp_active ||
  224             ((is >> 16) & ~SILI_PREG_IS_CMDCOMP)) {
DPRINTF(SILI_D_INTR, "%s: IS: 0x%08x (0x%b), PSS: %08x, "
  226                     "active: %08x\n", PORTNAME(sp), is, is >> 16, SILI_PFMT_IS,
pss_saved, sp->sp_active);
  228         }
  229 #endif
  231         /* Only interested in slot status bits. */
pss_saved &= SILI_PREG_PSS_ALL_SLOTS;
  234         if (is & SILI_PREG_IS_CMDERR) {
  235                 int                     err_slot, err_code;
u_int32_t               sactive = 0;
sili_pwrite(sp, SILI_PREG_IS, SILI_PREG_IS_CMDERR);
err_slot = SILI_PREG_PCS_ACTIVE(sili_pread(sp, SILI_PREG_PCS));
err_code = sili_pread(sp, SILI_PREG_CE);
ccb = &sp->sp_ccbs[err_slot];
  243                 switch (err_code) {
  244                 case SILI_PREG_CE_DEVICEERROR:
  245                 case SILI_PREG_CE_DATAFISERROR:
  246                         /* Extract error from command slot in LRAM. */
sili_pread_fis(sp, err_slot, &ccb->ccb_xa.rfis);
  248                         break;
  250                 case SILI_PREG_CE_SDBERROR:
  251                         /* No NCQ commands active?  Treat as a normal error. */
sactive = sili_pread(sp, SILI_PREG_SACT);/* XXX Pmult */
  253                         if (sactive == 0)
  254                                 break;
  256                         /* Extract real NCQ error slot & RFIS from log page. */
  257                         if (!sili_read_ncq_error(sp, &err_slot)) {
  258                                 /* got real err_slot */
ccb = &sp->sp_ccbs[err_slot];
  260                                 break;
  261                         }
  263                         /* failed to get error or not NCQ */
  265                         /* FALLTHROUGH */
  266                 default:
  267                         /* All other error types are fatal. */
printf("%s: fatal error (%d), aborting active slots "
  269                             "(%08x) and resetting device.\n", PORTNAME(sp),
err_code, pss_saved);
  271                         while (pss_saved) {
slot = ffs(pss_saved) - 1;
pss_saved &= ~(1 << slot);
ccb = &sp->sp_ccbs[slot];
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
ccb->ccb_xa.state = ATA_S_ERROR;
  278                         }
need_restart = SILI_PREG_PCS_DEVRESET;
  280                         goto fatal;
  281                 }
DPRINTF(SILI_D_VERBOSE, "%s: %serror, code %d, slot %d, "
  284                     "active %08x\n", PORTNAME(sp), sactive ? "NCQ " : "",
err_code, err_slot, sp->sp_active);
  287                 /* Clear the failed commmand in saved PSS so cmd_done runs. */
pss_saved &= ~(1 << err_slot);
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
ccb->ccb_xa.state = ATA_S_ERROR;
need_restart = SILI_PREG_PCS_PORTINIT;
  294         }
fatal:
  297         /* Process command timeout request only if command is still active. */
  298         if (timeout_slot >= 0 && (pss_saved & (1 << timeout_slot))) {
DPRINTF(SILI_D_VERBOSE, "%s: timing out slot %d, active %08x\n",
PORTNAME(sp), timeout_slot, sp->sp_active);
  302                 /* Clear the failed commmand in saved PSS so cmd_done runs. */
pss_saved &= ~(1 << timeout_slot);
ccb = &sp->sp_ccbs[timeout_slot];
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
ccb->ccb_xa.state = ATA_S_TIMEOUT;
  309                 /* Reset device to abort all commands (including this one). */
need_restart = SILI_PREG_PCS_DEVRESET;
  311         }
  313         /* Command slot is complete if its bit in PSS is 0 but 1 in active. */
pss_masked = ~pss_saved & sp->sp_active;
  315         while (pss_masked) {
slot = ffs(pss_masked) - 1;
ccb = &sp->sp_ccbs[slot];
pss_masked &= ~(1 << slot);
DPRINTF(SILI_D_INTR, "%s: slot %d is complete%s\n",
PORTNAME(sp), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
  322                     " (error)" : (ccb->ccb_xa.state == ATA_S_TIMEOUT ?
  323                     " (timeout)" : ""));
sili_ata_cmd_done(ccb, need_restart);
processed |= 1 << slot;
  328         }
  330         if (need_restart) {
  331                 /* Re-enable transfers on port. */
sili_pwrite(sp, SILI_PREG_PCS, need_restart);
  333                 if (!sili_pwait_eq(sp, SILI_PREG_PCS, SILI_PREG_PCS_PORTRDY,
SILI_PREG_PCS_PORTRDY, 1000)) {
printf("%s: couldn't restart port after error\n",
PORTNAME(sp));
  337                 }
  339                 /* Restart CCBs in the order they were originally queued. */
pss_masked = pss_saved;
TAILQ_FOREACH(ccb, &sp->sp_active_ccbs, ccb_entry) {
DPRINTF(SILI_D_VERBOSE, "%s: restarting slot %d "
  343                             "after error, state %02x\n", PORTNAME(sp),
ccb->ccb_xa.tag, ccb->ccb_xa.state);
  345                         if (!(pss_masked & (1 << ccb->ccb_xa.tag))) {
panic("sili_intr: slot %d not active in "
  347                                     "pss_masked: %08x, state %02x",
ccb->ccb_xa.tag, pss_masked,
ccb->ccb_xa.state);
  350                         }
pss_masked &= ~(1 << ccb->ccb_xa.tag);
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
sili_post_indirect(sp, ccb);
  355                 }
KASSERT(pss_masked == 0);
  358                 /*
  359                  * Finally, run atascsi completion for any finished CCBs.  If
  360                  * we had run these during cmd_done above, any ccbs that their
  361                  * completion generated would have been activated out of order.
  362                  */
  363                 while ((ccb = TAILQ_FIRST(&sp->sp_deferred_ccbs)) != NULL) {
TAILQ_REMOVE(&sp->sp_deferred_ccbs, ccb, ccb_entry);
KASSERT(ccb->ccb_xa.state == ATA_S_COMPLETE ||
ccb->ccb_xa.state == ATA_S_ERROR ||
ccb->ccb_xa.state == ATA_S_TIMEOUT);
ccb->ccb_xa.complete(&ccb->ccb_xa);
  370                 }
  371         }
  373         return (processed);
  374 }
  376 int
sili_intr(void *arg)
  378 {
  379         struct sili_softc               *sc = arg;
u_int32_t                       is;
  381         int                             port;
is = sili_read(sc, SILI_REG_GIS);
  384         if (is == 0)
  385                 return (0);
sili_write(sc, SILI_REG_GIS, is);
DPRINTF(SILI_D_INTR, "sili_intr, GIS: %08x\n", is);
  389         while (is & SILI_REG_GIS_PIS_MASK) {
port = ffs(is) - 1;
sili_port_intr(&sc->sc_ports[port], -1);
is &= ~(1 << port);
  393         }
  395         return (1);
  396 }
  398 int
sili_ports_alloc(struct sili_softc *sc)
  400 {
  401         struct sili_port                *sp;
  402         int                             i;
sc->sc_ports = malloc(sizeof(struct sili_port) * sc->sc_nports,
M_DEVBUF, M_WAITOK);
bzero(sc->sc_ports, sizeof(struct sili_port) * sc->sc_nports);
  408         for (i = 0; i < sc->sc_nports; i++) {
sp = &sc->sc_ports[i];
sp->sp_sc = sc;
  412 #ifdef SILI_DEBUG
snprintf(sp->sp_name, sizeof(sp->sp_name), "%s.%d",
DEVNAME(sc), i);
  415 #endif
  416                 if (bus_space_subregion(sc->sc_iot_port, sc->sc_ioh_port,
SILI_PORT_OFFSET(i), SILI_PORT_SIZE, &sp->sp_ioh) != 0) {
printf("%s: unable to create register window "
  419                             "for port %d\n", DEVNAME(sc), i);
  420                         goto freeports;
  421                 }
  422         }
  424         return (0);
freeports:
  427         /* bus_space(9) says subregions dont have to be freed */
free(sp, M_DEVBUF);
sc->sc_ports = NULL;
  430         return (1);
  431 }
  433 void
sili_ports_free(struct sili_softc *sc)
  435 {
  436         struct sili_port                *sp;
  437         int                             i;
  439         for (i = 0; i < sc->sc_nports; i++) {
sp = &sc->sc_ports[i];
  442                 if (sp->sp_ccbs != NULL)
sili_ccb_free(sp);
  444         }
  446         /* bus_space(9) says subregions dont have to be freed */
free(sc->sc_ports, M_DEVBUF);
sc->sc_ports = NULL;
  449 }
  451 int
sili_ccb_alloc(struct sili_port *sp)
  453 {
  454         struct sili_softc               *sc = sp->sp_sc;
  455         struct sili_ccb                 *ccb;
  456         struct sili_prb                 *prb;
  457         int                             i;
TAILQ_INIT(&sp->sp_free_ccbs);
TAILQ_INIT(&sp->sp_active_ccbs);
TAILQ_INIT(&sp->sp_deferred_ccbs);
sp->sp_ccbs = malloc(sizeof(struct sili_ccb) * SILI_MAX_CMDS,
M_DEVBUF, M_WAITOK);
sp->sp_cmds = sili_dmamem_alloc(sc, SILI_CMD_LEN * SILI_MAX_CMDS,
SILI_PRB_ALIGN);
  467         if (sp->sp_cmds == NULL)
  468                 goto free_ccbs;
sp->sp_scratch = sili_dmamem_alloc(sc, SILI_SCRATCH_LEN, PAGE_SIZE);
  470         if (sp->sp_scratch == NULL)
  471                 goto free_cmds;
bzero(sp->sp_ccbs, sizeof(struct sili_ccb) * SILI_MAX_CMDS);
  475         for (i = 0; i < SILI_MAX_CMDS; i++) {
ccb = &sp->sp_ccbs[i];
ccb->ccb_port = sp;
ccb->ccb_cmd = SILI_DMA_KVA(sp->sp_cmds) + i * SILI_CMD_LEN;
ccb->ccb_cmd_dva = SILI_DMA_DVA(sp->sp_cmds) + i * SILI_CMD_LEN;
  480                 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, SILI_DMA_SEGS,
MAXPHYS, 0, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
  482                     &ccb->ccb_dmamap) != 0)
  483                         goto free_scratch;
prb = ccb->ccb_cmd;
ccb->ccb_xa.fis = (struct ata_fis_h2d *)&prb->fis;
ccb->ccb_xa.packetcmd = ((struct sili_prb_packet *)prb)->cdb;
ccb->ccb_xa.tag = i;
ccb->ccb_xa.ata_put_xfer = sili_ata_put_xfer;
ccb->ccb_xa.state = ATA_S_COMPLETE;
sili_put_ccb(ccb);
  493         }
  495         return (0);
free_scratch:
sili_dmamem_free(sc, sp->sp_scratch);
free_cmds:
sili_dmamem_free(sc, sp->sp_cmds);
free_ccbs:
sili_ccb_free(sp);
  503         return (1);
  504 }
  506 void
sili_ccb_free(struct sili_port *sp)
  508 {
  509         struct sili_softc               *sc = sp->sp_sc;
  510         struct sili_ccb                 *ccb;
  512         while ((ccb = sili_get_ccb(sp)) != NULL)
bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
free(sp->sp_ccbs, M_DEVBUF);
sp->sp_ccbs = NULL;
  517 }
  519 struct sili_ccb *
sili_get_ccb(struct sili_port *sp)
  521 {
  522         struct sili_ccb                 *ccb;
ccb = TAILQ_FIRST(&sp->sp_free_ccbs);
  525         if (ccb != NULL) {
KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
TAILQ_REMOVE(&sp->sp_free_ccbs, ccb, ccb_entry);
ccb->ccb_xa.state = ATA_S_SETUP;
  529         }
  531         return (ccb);
  532 }
  534 void
sili_put_ccb(struct sili_ccb *ccb)
  536 {
  537         struct sili_port                *sp = ccb->ccb_port;
  539 #ifdef DIAGNOSTIC
  540         if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
ccb->ccb_xa.state != ATA_S_TIMEOUT &&
ccb->ccb_xa.state != ATA_S_ERROR) {
printf("%s: invalid ata_xfer state %02x in sili_put_ccb, "
  544                     "slot %d\n", PORTNAME(sp), ccb->ccb_xa.state,
ccb->ccb_xa.tag);
  546         }
  547 #endif
ccb->ccb_xa.state = ATA_S_PUT;
TAILQ_INSERT_TAIL(&sp->sp_free_ccbs, ccb, ccb_entry);
  551 }
  553 struct sili_dmamem *
sili_dmamem_alloc(struct sili_softc *sc, bus_size_t size, bus_size_t align)
  555 {
  556         struct sili_dmamem              *sdm;
  557         int                             nsegs;
sdm = malloc(sizeof(struct sili_dmamem), M_DEVBUF, M_WAITOK);
bzero(sdm, sizeof(struct sili_dmamem));
sdm->sdm_size = size;
  563         if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &sdm->sdm_map) != 0)
  565                 goto sdmfree;
  567         if (bus_dmamem_alloc(sc->sc_dmat, size, align, 0, &sdm->sdm_seg,
  568             1, &nsegs, BUS_DMA_NOWAIT) != 0)
  569                 goto destroy;
  571         if (bus_dmamem_map(sc->sc_dmat, &sdm->sdm_seg, nsegs, size,
  572             &sdm->sdm_kva, BUS_DMA_NOWAIT) != 0)
  573                 goto free;
  575         if (bus_dmamap_load(sc->sc_dmat, sdm->sdm_map, sdm->sdm_kva, size,
NULL, BUS_DMA_NOWAIT) != 0)
  577                 goto unmap;
bzero(sdm->sdm_kva, size);
  581         return (sdm);
unmap:
bus_dmamem_unmap(sc->sc_dmat, sdm->sdm_kva, size);
free:
bus_dmamem_free(sc->sc_dmat, &sdm->sdm_seg, 1);
destroy:
bus_dmamap_destroy(sc->sc_dmat, sdm->sdm_map);
sdmfree:
free(sdm, M_DEVBUF);
  592         return (NULL);
  593 }
  595 void
sili_dmamem_free(struct sili_softc *sc, struct sili_dmamem *sdm)
  597 {
bus_dmamap_unload(sc->sc_dmat, sdm->sdm_map);
bus_dmamem_unmap(sc->sc_dmat, sdm->sdm_kva, sdm->sdm_size);
bus_dmamem_free(sc->sc_dmat, &sdm->sdm_seg, 1);
bus_dmamap_destroy(sc->sc_dmat, sdm->sdm_map);
free(sdm, M_DEVBUF);
  603 }
u_int32_t
sili_read(struct sili_softc *sc, bus_size_t r)
  607 {
u_int32_t                       rv;
bus_space_barrier(sc->sc_iot_global, sc->sc_ioh_global, r, 4,
BUS_SPACE_BARRIER_READ);
rv = bus_space_read_4(sc->sc_iot_global, sc->sc_ioh_global, r);
  614         return (rv);
  615 }
  617 void
sili_write(struct sili_softc *sc, bus_size_t r, u_int32_t v)
  619 {
bus_space_write_4(sc->sc_iot_global, sc->sc_ioh_global, r, v);
bus_space_barrier(sc->sc_iot_global, sc->sc_ioh_global, r, 4,
BUS_SPACE_BARRIER_WRITE);
  623 }
u_int32_t
sili_pread(struct sili_port *sp, bus_size_t r)
  627 {
u_int32_t                       rv;
bus_space_barrier(sp->sp_sc->sc_iot_port, sp->sp_ioh, r, 4,
BUS_SPACE_BARRIER_READ);
rv = bus_space_read_4(sp->sp_sc->sc_iot_port, sp->sp_ioh, r);
  634         return (rv);
  635 }
  637 void
sili_pwrite(struct sili_port *sp, bus_size_t r, u_int32_t v)
  639 {
bus_space_write_4(sp->sp_sc->sc_iot_port, sp->sp_ioh, r, v);
bus_space_barrier(sp->sp_sc->sc_iot_port, sp->sp_ioh, r, 4,
BUS_SPACE_BARRIER_WRITE);
  643 }
  645 int
sili_pwait_eq(struct sili_port *sp, bus_size_t r, u_int32_t mask,
u_int32_t value, int timeout)
  648 {
  649         while ((sili_pread(sp, r) & mask) != value) {
  650                 if (timeout == 0)
  651                         return (0);
delay(1000);
timeout--;
  655         }
  657         return (1);
  658 }
  660 int
sili_pwait_ne(struct sili_port *sp, bus_size_t r, u_int32_t mask,
u_int32_t value, int timeout)
  663 {
  664         while ((sili_pread(sp, r) & mask) == value) {
  665                 if (timeout == 0)
  666                         return (0);
delay(1000);
timeout--;
  670         }
  672         return (1);
  673 }
  675 void
sili_post_direct(struct sili_port *sp, u_int slot, void *buf, size_t buflen)
  677 {
bus_size_t                      r = SILI_PREG_SLOT(slot);
  680 #ifdef DIAGNOSTIC
  681         if (buflen != 64 && buflen != 128)
panic("sili_pcopy: buflen of %d is not 64 or 128", buflen);
  683 #endif
bus_space_write_raw_region_4(sp->sp_sc->sc_iot_port, sp->sp_ioh, r,
buf, buflen);
bus_space_barrier(sp->sp_sc->sc_iot_port, sp->sp_ioh, r, buflen,
BUS_SPACE_BARRIER_WRITE);
sili_pwrite(sp, SILI_PREG_FIFO, slot);
  691 }
  693 void
sili_pread_fis(struct sili_port *sp, u_int slot, struct ata_fis_d2h *fis)
  695 {
bus_size_t                      r = SILI_PREG_SLOT(slot) + 8;
bus_space_barrier(sp->sp_sc->sc_iot_port, sp->sp_ioh, r,
  699             sizeof(struct ata_fis_d2h), BUS_SPACE_BARRIER_READ);
bus_space_read_raw_region_4(sp->sp_sc->sc_iot_port, sp->sp_ioh, r,
fis, sizeof(struct ata_fis_d2h));
  702 }
  704 void
sili_post_indirect(struct sili_port *sp, struct sili_ccb *ccb)
  706 {
sili_pwrite(sp, SILI_PREG_CAR_LO(ccb->ccb_xa.tag),
  708             (u_int32_t)ccb->ccb_cmd_dva);
sili_pwrite(sp, SILI_PREG_CAR_HI(ccb->ccb_xa.tag),
  710             (u_int32_t)(ccb->ccb_cmd_dva >> 32));
  711 }
u_int32_t
sili_signature(struct sili_port *sp, u_int slot)
  715 {
u_int32_t                       sig_hi, sig_lo;
sig_hi = sili_pread(sp, SILI_PREG_SIG_HI(slot));
sig_hi <<= SILI_PREG_SIG_HI_SHIFT;
sig_lo = sili_pread(sp, SILI_PREG_SIG_LO(slot));
sig_lo &= SILI_PREG_SIG_LO_MASK;
  723         return (sig_hi | sig_lo);
  724 }
  726 int
sili_ata_probe(void *xsc, int port)
  728 {
  729         struct sili_softc               *sc = xsc;
  730         struct sili_port                *sp = &sc->sc_ports[port];
  731         struct sili_prb_softreset       sreset;
u_int32_t                       signature;
  733         int                             port_type;
sili_pwrite(sp, SILI_PREG_PCC, SILI_PREG_PCC_PORTRESET);
sili_pwrite(sp, SILI_PREG_PCC, SILI_PREG_PCC_A32B);
  738         if (!sili_pwait_eq(sp, SILI_PREG_SSTS, SATA_SStatus_DET,
SATA_SStatus_DET_DEV, 1000))
  740                 return (ATA_PORT_T_NONE);
DPRINTF(SILI_D_VERBOSE, "%s: SSTS 0x%08x\n", PORTNAME(sp),
sili_pread(sp, SILI_PREG_SSTS));
bzero(&sreset, sizeof(sreset));
sreset.control = htole16(SILI_PRB_SOFT_RESET | SILI_PRB_INTERRUPT_MASK);
  747         /* XXX sreset fis pmp field */
  749         /* we use slot 0 */
sili_post_direct(sp, 0, &sreset, sizeof(sreset));
  751         if (!sili_pwait_eq(sp, SILI_PREG_PSS, (1 << 0), 0, 1000)) {
DPRINTF(SILI_D_VERBOSE, "%s: timed out while waiting for soft "
  753                     "reset\n", PORTNAME(sp));
  754                 return (ATA_PORT_T_NONE);
  755         }
  757         /* Read device signature from command slot. */
signature = sili_signature(sp, 0);
DPRINTF(SILI_D_VERBOSE, "%s: signature 0x%08x\n", PORTNAME(sp),
signature);
  763         switch (signature) {
  764         case SATA_SIGNATURE_DISK:
port_type = ATA_PORT_T_DISK;
  766                 break;
  767         case SATA_SIGNATURE_ATAPI:
port_type = ATA_PORT_T_ATAPI;
  769                 break;
  770         case SATA_SIGNATURE_PORT_MULTIPLIER:
  771         default:
  772                 return (ATA_PORT_T_NONE);
  773         }
  775         /* allocate port resources */
  776         if (sili_ccb_alloc(sp) != 0)
  777                 return (ATA_PORT_T_NONE);
  779         /* enable port interrupts */
sili_write(sc, SILI_REG_GC, sili_read(sc, SILI_REG_GC) | 1 << port);
sili_pwrite(sp, SILI_PREG_IES, SILI_PREG_IE_CMDERR |
SILI_PREG_IE_CMDCOMP);
  784         return (port_type);
  785 }
  787 int
sili_ata_cmd(struct ata_xfer *xa)
  789 {
  790         struct sili_ccb                 *ccb = (struct sili_ccb *)xa;
  791         struct sili_port                *sp = ccb->ccb_port;
  792         struct sili_softc               *sc = sp->sp_sc;
  793         struct sili_prb_ata             *ata;
  794         struct sili_prb_packet          *atapi;
  795         struct sili_sge                 *sgl;
  796         int                             sgllen;
  797         int                             s;
KASSERT(xa->state == ATA_S_SETUP);
  801         if (xa->flags & ATA_F_PACKET) {
atapi = ccb->ccb_cmd;
  804                 if (xa->flags & ATA_F_WRITE)
atapi->control = htole16(SILI_PRB_PACKET_WRITE);
  806                 else
atapi->control = htole16(SILI_PRB_PACKET_READ);
sgl = atapi->sgl;
sgllen = sizeofa(atapi->sgl);
  811         } else {
ata = ccb->ccb_cmd;
ata->control = 0;
sgl = ata->sgl;
sgllen = sizeofa(ata->sgl);
  818         }
  820         if (sili_load(ccb, sgl, sgllen) != 0)
  821                 goto failcmd;
bus_dmamap_sync(sc->sc_dmat, SILI_DMA_MAP(sp->sp_cmds),
xa->tag * SILI_CMD_LEN, SILI_CMD_LEN, BUS_DMASYNC_PREWRITE);
timeout_set(&xa->stimeout, sili_ata_cmd_timeout, ccb);
xa->state = ATA_S_PENDING;
  830         if (xa->flags & ATA_F_POLL) {
sili_poll(ccb, xa->timeout, sili_ata_cmd_timeout);
  832                 return (ATA_COMPLETE);
  833         }
timeout_add(&xa->stimeout, (xa->timeout * hz) / 1000);
s = splbio();
sili_start(sp, ccb);
splx(s);
  840         return (ATA_QUEUED);
failcmd:
s = splbio();
xa->state = ATA_S_ERROR;
xa->complete(xa);
splx(s);
  847         return (ATA_ERROR);
  848 }
  850 void
sili_ata_cmd_done(struct sili_ccb *ccb, int defer_completion)
  852 {
  853         struct sili_port                *sp = ccb->ccb_port;
  854         struct sili_softc               *sc = sp->sp_sc;
  855         struct ata_xfer                 *xa = &ccb->ccb_xa;
splassert(IPL_BIO);
timeout_del(&xa->stimeout);
bus_dmamap_sync(sc->sc_dmat, SILI_DMA_MAP(sp->sp_cmds),
xa->tag * SILI_CMD_LEN, SILI_CMD_LEN, BUS_DMASYNC_POSTWRITE);
sili_unload(ccb);
TAILQ_REMOVE(&sp->sp_active_ccbs, ccb, ccb_entry);
sp->sp_active &= ~(1 << xa->tag);
  869         if (xa->state == ATA_S_ONCHIP)
xa->state = ATA_S_COMPLETE;
  871 #ifdef DIAGNOSTIC
  872         else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
printf("%s: invalid ata_xfer state %02x in sili_ata_cmd_done, "
  874                     "slot %d\n", PORTNAME(sp), xa->state, xa->tag);
  875 #endif
  876         if (defer_completion)
TAILQ_INSERT_TAIL(&sp->sp_deferred_ccbs, ccb, ccb_entry);
  878         else if (xa->state == ATA_S_COMPLETE)
xa->complete(xa);
  880 #ifdef DIAGNOSTIC
  881         else
printf("%s: completion not deferred, but xa->state is %02x?\n",
PORTNAME(sp), xa->state);
  884 #endif
  885 }
  887 void
sili_ata_cmd_timeout(void *xccb)
  889 {
  890         struct sili_ccb                 *ccb = xccb;
  891         struct sili_port                *sp = ccb->ccb_port;
  892         int                             s;
s = splbio();
sili_port_intr(sp, ccb->ccb_xa.tag);
splx(s);
  897 }
  899 int
sili_load(struct sili_ccb *ccb, struct sili_sge *sgl, int sgllen)
  901 {
  902         struct sili_port                *sp = ccb->ccb_port;
  903         struct sili_softc               *sc = sp->sp_sc;
  904         struct ata_xfer                 *xa = &ccb->ccb_xa;
  905         struct sili_sge                 *nsge = sgl, *ce = NULL;
bus_dmamap_t                    dmap = ccb->ccb_dmamap;
u_int64_t                       addr;
  908         int                             error;
  909         int                             i;
  911         if (xa->datalen == 0)
  912                 return (0);
error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
  915             (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
  916         if (error != 0) {
printf("%s: error %d loading dmamap\n", PORTNAME(sp), error);
  918                 return (1);
  919         }
  921         if (dmap->dm_nsegs > sgllen)
ce = &sgl[sgllen - 1];
  924         for (i = 0; i < dmap->dm_nsegs; i++) {
  925                 if (nsge == ce) {
nsge++;
addr = ccb->ccb_cmd_dva;
addr += ((u_int8_t *)nsge - (u_int8_t *)ccb->ccb_cmd);
ce->addr_lo = htole32((u_int32_t)addr);
ce->addr_hi = htole32((u_int32_t)(addr >> 32));
ce->flags = htole32(SILI_SGE_LNK);
  935                         if ((dmap->dm_nsegs - i) > SILI_SGT_SGLLEN)
ce += SILI_SGT_SGLLEN;
  937                         else
ce = NULL;
  939                 }
sgl = nsge;
addr = dmap->dm_segs[i].ds_addr;
sgl->addr_lo = htole32((u_int32_t)addr);
sgl->addr_hi = htole32((u_int32_t)(addr >> 32));
sgl->data_count = htole32(dmap->dm_segs[i].ds_len);
sgl->flags = 0;
nsge++;
  950         }
sgl->flags |= htole32(SILI_SGE_TRM);
bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
  954             (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
BUS_DMASYNC_PREWRITE);
  957         return (0);
  958 }
  960 void
sili_unload(struct sili_ccb *ccb)
  962 {
  963         struct sili_port                *sp = ccb->ccb_port;
  964         struct sili_softc               *sc = sp->sp_sc;
  965         struct ata_xfer                 *xa = &ccb->ccb_xa;
bus_dmamap_t                    dmap = ccb->ccb_dmamap;
  968         if (xa->datalen == 0)
  969                 return;
bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
  972             (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, dmap);
  976         if (xa->flags & ATA_F_READ)
xa->resid = xa->datalen - sili_pread(sp,
SILI_PREG_RX_COUNT(xa->tag));
  979         else
xa->resid = 0;
  981 }
  983 int
sili_poll(struct sili_ccb *ccb, int timeout, void (*timeout_fn)(void *))
  985 {
  986         struct sili_port                *sp = ccb->ccb_port;
  987         int                             s;
s = splbio();
sili_start(sp, ccb);
  991         do {
  992                 if (sili_port_intr(sp, -1) & (1 << ccb->ccb_xa.tag)) {
splx(s);
  994                         return (0);
  995                 }
delay(1000);
  998         } while (--timeout > 0);
 1000         /* Run timeout while at splbio, otherwise sili_intr could interfere. */
 1001         if (timeout_fn != NULL)
timeout_fn(ccb);
splx(s);
 1006         return (1);
 1007 }
 1009 void
sili_start(struct sili_port *sp, struct sili_ccb *ccb)
 1011 {
 1012         int                             slot = ccb->ccb_xa.tag;
splassert(IPL_BIO);
KASSERT(ccb->ccb_xa.state == ATA_S_PENDING);
TAILQ_INSERT_TAIL(&sp->sp_active_ccbs, ccb, ccb_entry);
sp->sp_active |= 1 << slot;
ccb->ccb_xa.state = ATA_S_ONCHIP;
sili_post_indirect(sp, ccb);
 1022 }
 1024 int
sili_read_ncq_error(struct sili_port *sp, int *err_slotp)
 1026 {
 1027         struct sili_softc               *sc = sp->sp_sc;
 1028         struct sili_prb_ata             read_10h;
u_int64_t                       addr;
 1030         struct ata_fis_h2d              *fis;
 1031         struct ata_log_page_10h         *log;
 1032         struct sili_ccb                 *ccb;
 1033         int                             rc;
sili_pwrite(sp, SILI_PREG_PCS, SILI_PREG_PCS_PORTINIT);
 1036         if (!sili_pwait_eq(sp, SILI_PREG_PCS, SILI_PREG_PCS_PORTRDY,
SILI_PREG_PCS_PORTRDY, 1000)) {
printf("%s: couldn't ready port during log page read\n",
PORTNAME(sp));
 1040                 return (1);
 1041         }
 1043         /* READ LOG EXT 10h into scratch space */
bzero(&read_10h, sizeof(read_10h));
read_10h.control = htole16(SILI_PRB_INTERRUPT_MASK);
addr = SILI_DMA_DVA(sp->sp_scratch);
read_10h.sgl[0].addr_lo = htole32((u_int32_t)addr);
read_10h.sgl[0].addr_hi = htole32((u_int32_t)(addr >> 32));
read_10h.sgl[0].data_count = htole32(512);
read_10h.sgl[0].flags = htole32(SILI_SGE_TRM);
fis = (struct ata_fis_h2d *)read_10h.fis;
fis->type = ATA_FIS_TYPE_H2D;
fis->flags = ATA_H2D_FLAGS_CMD; /* XXX fis pmp field */
fis->command = ATA_C_READ_LOG_EXT;
fis->lba_low = 0x10;            /* queued error log page (10h) */
fis->sector_count = 1;          /* number of sectors (1) */
fis->sector_count_exp = 0;
fis->lba_mid = 0;               /* starting offset */
fis->lba_mid_exp = 0;
fis->device = 0;
bus_dmamap_sync(sc->sc_dmat, SILI_DMA_MAP(sp->sp_scratch), 0,
 1065             512, BUS_DMASYNC_PREREAD);
 1067         /* issue read and poll for completion */
sili_post_direct(sp, 0, &read_10h, sizeof(read_10h));
rc = sili_pwait_eq(sp, SILI_PREG_PSS, (1 << 0), 0, 1000);
bus_dmamap_sync(sc->sc_dmat, SILI_DMA_MAP(sp->sp_scratch), 0,
 1072             512, BUS_DMASYNC_POSTREAD);
 1074         if (!rc) {
DPRINTF(SILI_D_VERBOSE, "%s: timed out while waiting for log "
 1076                     "page read\n", PORTNAME(sp));
 1077                 return (1);
 1078         }
 1080         /* Extract failed register set and tags from the scratch space. */
log = (struct ata_log_page_10h *)SILI_DMA_KVA(sp->sp_scratch);
 1082         if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
 1083                 /* Not queued bit was set - wasn't an NCQ error? */
printf("%s: read NCQ error page, but not an NCQ error?\n",
PORTNAME(sp));
 1086                 return (1);
 1087         }
 1089         /* Copy back the log record as a D2H register FIS. */
 1090         *err_slotp = log->err_regs.type & ATA_LOG_10H_TYPE_TAG_MASK;
ccb = &sp->sp_ccbs[*err_slotp];
memcpy(&ccb->ccb_xa.rfis, &log->err_regs, sizeof(struct ata_fis_d2h));
ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
ccb->ccb_xa.rfis.flags = 0;
 1097         return (0);
 1098 }
 1100 struct ata_xfer *
sili_ata_get_xfer(void *xsc, int port)
 1102 {
 1103         struct sili_softc               *sc = xsc;
 1104         struct sili_port                *sp = &sc->sc_ports[port];
 1105         struct sili_ccb                 *ccb;
ccb = sili_get_ccb(sp);
 1108         if (ccb == NULL) {
printf("sili_ata_get_xfer: NULL ccb\n");
 1110                 return (NULL);
 1111         }
bzero(ccb->ccb_cmd, SILI_CMD_LEN);
 1115         return ((struct ata_xfer *)ccb);
 1116 }
 1118 void
sili_ata_put_xfer(struct ata_xfer *xa)
 1120 {
 1121         struct sili_ccb                 *ccb = (struct sili_ccb *)xa;
sili_put_ccb(ccb);
 1124 }