root/dev/pci/ahci.c

DEFINITIONS

1. ahci_lookup_device
2. ahci_no_match
3. ahci_vt8251_attach
4. ahci_pci_match
5. ahci_pci_attach
6. ahci_map_regs
7. ahci_unmap_regs
8. ahci_map_intr
9. ahci_unmap_intr
10. ahci_init
11. ahci_port_alloc
12. ahci_port_free
13. ahci_port_start
14. ahci_port_stop
15. ahci_port_clo
16. ahci_port_softreset
17. ahci_port_portreset
18. ahci_load_prdt
19. ahci_unload_prdt
20. ahci_poll
21. ahci_start
22. ahci_issue_pending_ncq_commands
23. ahci_issue_pending_commands
24. ahci_intr
25. ahci_port_intr
26. ahci_get_ccb
27. ahci_put_ccb
28. ahci_get_err_ccb
29. ahci_put_err_ccb
30. ahci_port_read_ncq_error
31. ahci_dmamem_alloc
32. ahci_dmamem_free
33. ahci_read
34. ahci_write
35. ahci_wait_ne
36. ahci_pread
37. ahci_pwrite
38. ahci_pwait_eq
39. ahci_ata_probe
40. ahci_ata_get_xfer
41. ahci_ata_put_xfer
42. ahci_ata_cmd
43. ahci_ata_cmd_done
44. ahci_ata_cmd_timeout
45. ahci_empty_done

    1 /*      $OpenBSD: ahci.c,v 1.128 2007/07/03 22:33:20 dlg Exp $ */
    3 /*
    4  * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
    5  *
    6  * Permission to use, copy, modify, and distribute this software for any
    7  * purpose with or without fee is hereby granted, provided that the above
    8  * copyright notice and this permission notice appear in all copies.
    9  *
   10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
   12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
   13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
   14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
   15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
   16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
   17  */
   19 #include <sys/param.h>
   20 #include <sys/systm.h>
   21 #include <sys/buf.h>
   22 #include <sys/kernel.h>
   23 #include <sys/malloc.h>
   24 #include <sys/device.h>
   25 #include <sys/proc.h>
   26 #include <sys/queue.h>
   28 #include <machine/bus.h>
   30 #include <dev/pci/pcireg.h>
   31 #include <dev/pci/pcivar.h>
   32 #include <dev/pci/pcidevs.h>
   34 #include <dev/ata/atascsi.h>
   36 /* change to AHCI_DEBUG for dmesg spam */
   37 #define NO_AHCI_DEBUG
   39 #ifdef AHCI_DEBUG
   40 #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
   41     while (0)
   42 #define AHCI_D_TIMEOUT          0x00
   43 #define AHCI_D_VERBOSE          0x01
   44 #define AHCI_D_INTR             0x02
   45 #define AHCI_D_XFER             0x08
   46 int ahcidebug = AHCI_D_VERBOSE;
   47 #else
   48 #define DPRINTF(m, f...)
   49 #endif
   51 #define AHCI_PCI_BAR            0x24
   52 #define AHCI_PCI_INTERFACE      0x01
   54 #define AHCI_REG_CAP            0x000 /* HBA Capabilities */
   55 #define  AHCI_REG_CAP_NP(_r)            (((_r) & 0x1f)+1) /* Number of Ports */
   56 #define  AHCI_REG_CAP_SXS               (1<<5) /* External SATA */
   57 #define  AHCI_REG_CAP_EMS               (1<<6) /* Enclosure Mgmt */
   58 #define  AHCI_REG_CAP_CCCS              (1<<7) /* Cmd Coalescing */
   59 #define  AHCI_REG_CAP_NCS(_r)           ((((_r) & 0x1f00)>>8)+1) /* NCmds*/
   60 #define  AHCI_REG_CAP_PSC               (1<<13) /* Partial State Capable */
   61 #define  AHCI_REG_CAP_SSC               (1<<14) /* Slumber State Capable */
   62 #define  AHCI_REG_CAP_PMD               (1<<15) /* PIO Multiple DRQ Block */
   63 #define  AHCI_REG_CAP_FBSS              (1<<16) /* FIS-Based Switching */
   64 #define  AHCI_REG_CAP_SPM               (1<<17) /* Port Multiplier */
   65 #define  AHCI_REG_CAP_SAM               (1<<18) /* AHCI Only mode */
   66 #define  AHCI_REG_CAP_SNZO              (1<<19) /* Non Zero DMA Offsets */
   67 #define  AHCI_REG_CAP_ISS               (0xf<<20) /* Interface Speed Support */
   68 #define  AHCI_REG_CAP_ISS_G1            (0x1<<20) /* Gen 1 (1.5 Gbps) */
   69 #define  AHCI_REG_CAP_ISS_G1_2          (0x2<<20) /* Gen 1 and 2 (3 Gbps) */
   70 #define  AHCI_REG_CAP_SCLO              (1<<24) /* Cmd List Override */
   71 #define  AHCI_REG_CAP_SAL               (1<<25) /* Activity LED */
   72 #define  AHCI_REG_CAP_SALP              (1<<26) /* Aggressive Link Pwr Mgmt */
   73 #define  AHCI_REG_CAP_SSS               (1<<27) /* Staggered Spinup */
   74 #define  AHCI_REG_CAP_SMPS              (1<<28) /* Mech Presence Switch */
   75 #define  AHCI_REG_CAP_SSNTF             (1<<29) /* SNotification Register */
   76 #define  AHCI_REG_CAP_SNCQ              (1<<30) /* Native Cmd Queuing */
   77 #define  AHCI_REG_CAP_S64A              (1<<31) /* 64bit Addressing */
   78 #define  AHCI_FMT_CAP           "\020" "\040S64A" "\037NCQ" "\036SSNTF" \
   79                                     "\035SMPS" "\034SSS" "\033SALP" "\032SAL" \
   80                                     "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" \
   81                                     "\021FBSS" "\020PMD" "\017SSC" "\016PSC" \
   82                                     "\010CCCS" "\007EMS" "\006SXS"
   83 #define AHCI_REG_GHC            0x004 /* Global HBA Control */
   84 #define  AHCI_REG_GHC_HR                (1<<0) /* HBA Reset */
   85 #define  AHCI_REG_GHC_IE                (1<<1) /* Interrupt Enable */
   86 #define  AHCI_REG_GHC_MRSM              (1<<2) /* MSI Revert to Single Msg */
   87 #define  AHCI_REG_GHC_AE                (1<<31) /* AHCI Enable */
   88 #define AHCI_FMT_GHC            "\020" "\040AE" "\003MRSM" "\002IE" "\001HR"
   89 #define AHCI_REG_IS             0x008 /* Interrupt Status */
   90 #define AHCI_REG_PI             0x00c /* Ports Implemented */
   91 #define AHCI_REG_VS             0x010 /* AHCI Version */
   92 #define  AHCI_REG_VS_0_95               0x00000905 /* 0.95 */
   93 #define  AHCI_REG_VS_1_0                0x00010000 /* 1.0 */
   94 #define  AHCI_REG_VS_1_1                0x00010100 /* 1.1 */
   95 #define AHCI_REG_CCC_CTL        0x014 /* Coalescing Control */
   96 #define  AHCI_REG_CCC_CTL_INT(_r)       (((_r) & 0xf8) >> 3) /* CCC INT slot */
   97 #define AHCI_REG_CCC_PORTS      0x018 /* Coalescing Ports */
   98 #define AHCI_REG_EM_LOC         0x01c /* Enclosure Mgmt Location */
   99 #define AHCI_REG_EM_CTL         0x020 /* Enclosure Mgmt Control */
  101 #define AHCI_PORT_REGION(_p)    (0x100 + ((_p) * 0x80))
  102 #define AHCI_PORT_SIZE          0x80
  104 #define AHCI_PREG_CLB           0x00 /* Cmd List Base Addr */
  105 #define AHCI_PREG_CLBU          0x04 /* Cmd List Base Hi Addr */
  106 #define AHCI_PREG_FB            0x08 /* FIS Base Addr */
  107 #define AHCI_PREG_FBU           0x0c /* FIS Base Hi Addr */
  108 #define AHCI_PREG_IS            0x10 /* Interrupt Status */
  109 #define  AHCI_PREG_IS_DHRS              (1<<0) /* Device to Host FIS */
  110 #define  AHCI_PREG_IS_PSS               (1<<1) /* PIO Setup FIS */
  111 #define  AHCI_PREG_IS_DSS               (1<<2) /* DMA Setup FIS */
  112 #define  AHCI_PREG_IS_SDBS              (1<<3) /* Set Device Bits FIS */
  113 #define  AHCI_PREG_IS_UFS               (1<<4) /* Unknown FIS */
  114 #define  AHCI_PREG_IS_DPS               (1<<5) /* Descriptor Processed */
  115 #define  AHCI_PREG_IS_PCS               (1<<6) /* Port Change */
  116 #define  AHCI_PREG_IS_DMPS              (1<<7) /* Device Mechanical Presence */
  117 #define  AHCI_PREG_IS_PRCS              (1<<22) /* PhyRdy Change */
  118 #define  AHCI_PREG_IS_IPMS              (1<<23) /* Incorrect Port Multiplier */
  119 #define  AHCI_PREG_IS_OFS               (1<<24) /* Overflow */
  120 #define  AHCI_PREG_IS_INFS              (1<<26) /* Interface Non-fatal Error */
  121 #define  AHCI_PREG_IS_IFS               (1<<27) /* Interface Fatal Error */
  122 #define  AHCI_PREG_IS_HBDS              (1<<28) /* Host Bus Data Error */
  123 #define  AHCI_PREG_IS_HBFS              (1<<29) /* Host Bus Fatal Error */
  124 #define  AHCI_PREG_IS_TFES              (1<<30) /* Task File Error */
  125 #define  AHCI_PREG_IS_CPDS              (1<<31) /* Cold Presence Detect */
  126 #define AHCI_PFMT_IS            "\20" "\040CPDS" "\037TFES" "\036HBFS" \
  127                                     "\035HBDS" "\034IFS" "\033INFS" "\031OFS" \
  128                                     "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" \
  129                                     "\007PCS" "\005UFS" "\004SDBS" "\003DSS" \
  130                                     "\002PSS" "\001DHRS"
  131 #define AHCI_PREG_IE            0x14 /* Interrupt Enable */
  132 #define  AHCI_PREG_IE_DHRE              (1<<0) /* Device to Host FIS */
  133 #define  AHCI_PREG_IE_PSE               (1<<1) /* PIO Setup FIS */
  134 #define  AHCI_PREG_IE_DSE               (1<<2) /* DMA Setup FIS */
  135 #define  AHCI_PREG_IE_SDBE              (1<<3) /* Set Device Bits FIS */
  136 #define  AHCI_PREG_IE_UFE               (1<<4) /* Unknown FIS */
  137 #define  AHCI_PREG_IE_DPE               (1<<5) /* Descriptor Processed */
  138 #define  AHCI_PREG_IE_PCE               (1<<6) /* Port Change */
  139 #define  AHCI_PREG_IE_DMPE              (1<<7) /* Device Mechanical Presence */
  140 #define  AHCI_PREG_IE_PRCE              (1<<22) /* PhyRdy Change */
  141 #define  AHCI_PREG_IE_IPME              (1<<23) /* Incorrect Port Multiplier */
  142 #define  AHCI_PREG_IE_OFE               (1<<24) /* Overflow */
  143 #define  AHCI_PREG_IE_INFE              (1<<26) /* Interface Non-fatal Error */
  144 #define  AHCI_PREG_IE_IFE               (1<<27) /* Interface Fatal Error */
  145 #define  AHCI_PREG_IE_HBDE              (1<<28) /* Host Bus Data Error */
  146 #define  AHCI_PREG_IE_HBFE              (1<<29) /* Host Bus Fatal Error */
  147 #define  AHCI_PREG_IE_TFEE              (1<<30) /* Task File Error */
  148 #define  AHCI_PREG_IE_CPDE              (1<<31) /* Cold Presence Detect */
  149 #define AHCI_PFMT_IE            "\20" "\040CPDE" "\037TFEE" "\036HBFE" \
  150                                     "\035HBDE" "\034IFE" "\033INFE" "\031OFE" \
  151                                     "\030IPME" "\027PRCE" "\010DMPE" "\007PCE" \
  152                                     "\006DPE" "\005UFE" "\004SDBE" "\003DSE" \
  153                                     "\002PSE" "\001DHRE"
  154 #define AHCI_PREG_CMD           0x18 /* Command and Status */
  155 #define  AHCI_PREG_CMD_ST               (1<<0) /* Start */
  156 #define  AHCI_PREG_CMD_SUD              (1<<1) /* Spin Up Device */
  157 #define  AHCI_PREG_CMD_POD              (1<<2) /* Power On Device */
  158 #define  AHCI_PREG_CMD_CLO              (1<<3) /* Command List Override */
  159 #define  AHCI_PREG_CMD_FRE              (1<<4) /* FIS Receive Enable */
  160 #define  AHCI_PREG_CMD_CCS(_r)          (((_r) >> 8) & 0x1f) /* Curr CmdSlot# */
  161 #define  AHCI_PREG_CMD_MPSS             (1<<13) /* Mech Presence State */
  162 #define  AHCI_PREG_CMD_FR               (1<<14) /* FIS Receive Running */
  163 #define  AHCI_PREG_CMD_CR               (1<<15) /* Command List Running */
  164 #define  AHCI_PREG_CMD_CPS              (1<<16) /* Cold Presence State */
  165 #define  AHCI_PREG_CMD_PMA              (1<<17) /* Port Multiplier Attached */
  166 #define  AHCI_PREG_CMD_HPCP             (1<<18) /* Hot Plug Capable */
  167 #define  AHCI_PREG_CMD_MPSP             (1<<19) /* Mech Presence Switch */
  168 #define  AHCI_PREG_CMD_CPD              (1<<20) /* Cold Presence Detection */
  169 #define  AHCI_PREG_CMD_ESP              (1<<21) /* External SATA Port */
  170 #define  AHCI_PREG_CMD_ATAPI            (1<<24) /* Device is ATAPI */
  171 #define  AHCI_PREG_CMD_DLAE             (1<<25) /* Drv LED on ATAPI Enable */
  172 #define  AHCI_PREG_CMD_ALPE             (1<<26) /* Aggro Pwr Mgmt Enable */
  173 #define  AHCI_PREG_CMD_ASP              (1<<27) /* Aggro Slumber/Partial */
  174 #define  AHCI_PREG_CMD_ICC              0xf0000000 /* Interface Comm Ctrl */
  175 #define  AHCI_PREG_CMD_ICC_SLUMBER      0x60000000
  176 #define  AHCI_PREG_CMD_ICC_PARTIAL      0x20000000
  177 #define  AHCI_PREG_CMD_ICC_ACTIVE       0x10000000
  178 #define  AHCI_PREG_CMD_ICC_IDLE         0x00000000
  179 #define  AHCI_PFMT_CMD          "\020" "\034ASP" "\033ALPE" "\032DLAE" \
  180                                     "\031ATAPI" "\026ESP" "\025CPD" "\024MPSP" \
  181                                     "\023HPCP" "\022PMA" "\021CPS" "\020CR" \
  182                                     "\017FR" "\016MPSS" "\005FRE" "\004CLO" \
  183                                     "\003POD" "\002SUD" "\001ST"
  184 #define AHCI_PREG_TFD           0x20 /* Task File Data*/
  185 #define  AHCI_PREG_TFD_STS              0xff
  186 #define  AHCI_PREG_TFD_STS_ERR          (1<<0)
  187 #define  AHCI_PREG_TFD_STS_DRQ          (1<<3)
  188 #define  AHCI_PREG_TFD_STS_BSY          (1<<7)
  189 #define  AHCI_PREG_TFD_ERR              0xff00
  190 #define AHCI_PFMT_TFD_STS       "\20" "\010BSY" "\004DRQ" "\001ERR"
  191 #define AHCI_PREG_SIG           0x24 /* Signature */
  192 #define AHCI_PREG_SSTS          0x28 /* SATA Status */
  193 #define  AHCI_PREG_SSTS_DET             0xf /* Device Detection */
  194 #define  AHCI_PREG_SSTS_DET_NONE        0x0
  195 #define  AHCI_PREG_SSTS_DET_DEV_NE      0x1
  196 #define  AHCI_PREG_SSTS_DET_DEV         0x3
  197 #define  AHCI_PREG_SSTS_DET_PHYOFFLINE  0x4
  198 #define  AHCI_PREG_SSTS_SPD             0xf0 /* Current Interface Speed */
  199 #define  AHCI_PREG_SSTS_SPD_NONE        0x00
  200 #define  AHCI_PREG_SSTS_SPD_GEN1        0x10
  201 #define  AHCI_PREG_SSTS_SPD_GEN2        0x20
  202 #define  AHCI_PREG_SSTS_IPM             0xf00 /* Interface Power Management */
  203 #define  AHCI_PREG_SSTS_IPM_NONE        0x000
  204 #define  AHCI_PREG_SSTS_IPM_ACTIVE      0x100
  205 #define  AHCI_PREG_SSTS_IPM_PARTIAL     0x200
  206 #define  AHCI_PREG_SSTS_IPM_SLUMBER     0x600
  207 #define AHCI_PREG_SCTL          0x2c /* SATA Control */
  208 #define  AHCI_PREG_SCTL_DET             0xf /* Device Detection */
  209 #define  AHCI_PREG_SCTL_DET_NONE        0x0
  210 #define  AHCI_PREG_SCTL_DET_INIT        0x1
  211 #define  AHCI_PREG_SCTL_DET_DISABLE     0x4
  212 #define  AHCI_PREG_SCTL_SPD             0xf0 /* Speed Allowed */
  213 #define  AHCI_PREG_SCTL_SPD_ANY         0x00
  214 #define  AHCI_PREG_SCTL_SPD_GEN1        0x10
  215 #define  AHCI_PREG_SCTL_SPD_GEN2        0x20
  216 #define  AHCI_PREG_SCTL_IPM             0xf00 /* Interface Power Management */
  217 #define  AHCI_PREG_SCTL_IPM_NONE        0x000
  218 #define  AHCI_PREG_SCTL_IPM_NOPARTIAL   0x100
  219 #define  AHCI_PREG_SCTL_IPM_NOSLUMBER   0x200
  220 #define  AHCI_PREG_SCTL_IPM_DISABLED    0x300
  221 #define AHCI_PREG_SERR          0x30 /* SATA Error */
  222 #define  AHCI_PREG_SERR_ERR(_r)         ((_r) & 0xffff)
  223 #define  AHCI_PREG_SERR_ERR_I           (1<<0) /* Recovered Data Integrity */
  224 #define  AHCI_PREG_SERR_ERR_M           (1<<1) /* Recovered Communications */
  225 #define  AHCI_PREG_SERR_ERR_T           (1<<8) /* Transient Data Integrity */
  226 #define  AHCI_PREG_SERR_ERR_C           (1<<9) /* Persistent Comm/Data */
  227 #define  AHCI_PREG_SERR_ERR_P           (1<<10) /* Protocol */
  228 #define  AHCI_PREG_SERR_ERR_E           (1<<11) /* Internal */
  229 #define  AHCI_PFMT_SERR_ERR     "\020" "\014E" "\013P" "\012C" "\011T" "\002M" \
  230                                     "\001I"
  231 #define  AHCI_PREG_SERR_DIAG(_r)        (((_r) >> 16) & 0xffff)
  232 #define  AHCI_PREG_SERR_DIAG_N          (1<<0) /* PhyRdy Change */
  233 #define  AHCI_PREG_SERR_DIAG_I          (1<<1) /* Phy Internal Error */
  234 #define  AHCI_PREG_SERR_DIAG_W          (1<<2) /* Comm Wake */
  235 #define  AHCI_PREG_SERR_DIAG_B          (1<<3) /* 10B to 8B Decode Error */
  236 #define  AHCI_PREG_SERR_DIAG_D          (1<<4) /* Disparity Error */
  237 #define  AHCI_PREG_SERR_DIAG_C          (1<<5) /* CRC Error */
  238 #define  AHCI_PREG_SERR_DIAG_H          (1<<6) /* Handshake Error */
  239 #define  AHCI_PREG_SERR_DIAG_S          (1<<7) /* Link Sequence Error */
  240 #define  AHCI_PREG_SERR_DIAG_T          (1<<8) /* Transport State Trans Err */
  241 #define  AHCI_PREG_SERR_DIAG_F          (1<<9) /* Unknown FIS Type */
  242 #define  AHCI_PREG_SERR_DIAG_X          (1<<10) /* Exchanged */
  243 #define  AHCI_PFMT_SERR_DIAG    "\020" "\013X" "\012F" "\011T" "\010S" "\007H" \
  244                                     "\006C" "\005D" "\004B" "\003W" "\002I" \
  245                                     "\001N"
  246 #define AHCI_PREG_SACT          0x34 /* SATA Active */
  247 #define AHCI_PREG_CI            0x38 /* Command Issue */
  248 #define  AHCI_PREG_CI_ALL_SLOTS 0xffffffff
  249 #define AHCI_PREG_SNTF          0x3c /* SNotification */
  251 struct ahci_cmd_hdr {
u_int16_t               flags;
  253 #define AHCI_CMD_LIST_FLAG_CFL          0x001f /* Command FIS Length */
  254 #define AHCI_CMD_LIST_FLAG_A            (1<<5) /* ATAPI */
  255 #define AHCI_CMD_LIST_FLAG_W            (1<<6) /* Write */
  256 #define AHCI_CMD_LIST_FLAG_P            (1<<7) /* Prefetchable */
  257 #define AHCI_CMD_LIST_FLAG_R            (1<<8) /* Reset */
  258 #define AHCI_CMD_LIST_FLAG_B            (1<<9) /* BIST */
  259 #define AHCI_CMD_LIST_FLAG_C            (1<<10) /* Clear Busy upon R_OK */
  260 #define AHCI_CMD_LIST_FLAG_PMP          0xf000 /* Port Multiplier Port */
u_int16_t               prdtl; /* sgl len */
u_int32_t               prdbc; /* transferred byte count */
u_int32_t               ctba_lo;
u_int32_t               ctba_hi;
u_int32_t               reserved[4];
  269 } __packed;
  271 struct ahci_rfis {
u_int8_t                dsfis[28];
u_int8_t                reserved1[4];
u_int8_t                psfis[24];
u_int8_t                reserved2[8];
u_int8_t                rfis[24];
u_int8_t                reserved3[4];
u_int8_t                sdbfis[4];
u_int8_t                ufis[64];
u_int8_t                reserved4[96];
  281 } __packed;
  283 struct ahci_prdt {
u_int32_t               dba_lo;
u_int32_t               dba_hi;
u_int32_t               reserved;
u_int32_t               flags;
  288 #define AHCI_PRDT_FLAG_INTR             (1<<31) /* interrupt on completion */
  289 } __packed;
  291 /* this makes ahci_cmd_table 512 bytes, supporting 128-byte alignment */
  292 #define AHCI_MAX_PRDT           24
  294 struct ahci_cmd_table {
u_int8_t                cfis[64];       /* Command FIS */
u_int8_t                acmd[16];       /* ATAPI Command */
u_int8_t                reserved[48];
  299         struct ahci_prdt        prdt[AHCI_MAX_PRDT];
  300 } __packed;
  302 #define AHCI_MAX_PORTS          32
  304 struct ahci_dmamem {
bus_dmamap_t            adm_map;
bus_dma_segment_t       adm_seg;
size_t                  adm_size;
caddr_t                 adm_kva;
  309 };
  310 #define AHCI_DMA_MAP(_adm)      ((_adm)->adm_map)
  311 #define AHCI_DMA_DVA(_adm)      ((_adm)->adm_map->dm_segs[0].ds_addr)
  312 #define AHCI_DMA_KVA(_adm)      ((void *)(_adm)->adm_kva)
  314 struct ahci_softc;
  315 struct ahci_port;
  317 struct ahci_ccb {
  318         /* ATA xfer associated with this CCB.  Must be 1st struct member. */
  319         struct ata_xfer         ccb_xa;
  321         int                     ccb_slot;
  322         struct ahci_port        *ccb_port;
bus_dmamap_t            ccb_dmamap;
  325         struct ahci_cmd_hdr     *ccb_cmd_hdr;
  326         struct ahci_cmd_table   *ccb_cmd_table;
  328         void                    (*ccb_done)(struct ahci_ccb *);
TAILQ_ENTRY(ahci_ccb)   ccb_entry;
  331 };
  333 struct ahci_port {
  334         struct ahci_softc       *ap_sc;
bus_space_handle_t      ap_ioh;
  337 #ifdef AHCI_COALESCE
  338         int                     ap_num;
  339 #endif
  341         struct ahci_rfis        *ap_rfis;
  342         struct ahci_dmamem      *ap_dmamem_rfis;
  344         struct ahci_dmamem      *ap_dmamem_cmd_list;
  345         struct ahci_dmamem      *ap_dmamem_cmd_table;
  347         volatile u_int32_t      ap_active;
  348         volatile u_int32_t      ap_active_cnt;
  349         volatile u_int32_t      ap_sactive;
  350         struct ahci_ccb         *ap_ccbs;
TAILQ_HEAD(, ahci_ccb)  ap_ccb_free;
TAILQ_HEAD(, ahci_ccb)  ap_ccb_pending;
u_int32_t               ap_state;
  356 #define AP_S_NORMAL                     0
  357 #define AP_S_FATAL_ERROR                1
  359         /* For error recovery. */
  360 #ifdef DIAGNOSTIC
  361         int                     ap_err_busy;
  362 #endif
u_int32_t               ap_err_saved_sactive;
u_int32_t               ap_err_saved_active;
u_int32_t               ap_err_saved_active_cnt;
u_int8_t                ap_err_scratch[512];
  369 #ifdef AHCI_DEBUG
  370         char                    ap_name[16];
  371 #define PORTNAME(_ap)   ((_ap)->ap_name)
  372 #else
  373 #define PORTNAME(_ap)   DEVNAME((_ap)->ap_sc)
  374 #endif
  375 };
  377 struct ahci_softc {
  378         struct device           sc_dev;
  380         void                    *sc_ih;
bus_space_tag_t         sc_iot;
bus_space_handle_t      sc_ioh;
bus_size_t              sc_ios;
bus_dma_tag_t           sc_dmat;
  387         int                     sc_flags;
  388 #define AHCI_F_NO_NCQ                   (1<<0)
u_int                   sc_ncmds;
  392         struct ahci_port        *sc_ports[AHCI_MAX_PORTS];
  394         struct atascsi          *sc_atascsi;
  396 #ifdef AHCI_COALESCE
u_int32_t               sc_ccc_mask;
u_int32_t               sc_ccc_ports;
u_int32_t               sc_ccc_ports_cur;
  400 #endif
  401 };
  402 #define DEVNAME(_s)             ((_s)->sc_dev.dv_xname)
  404 struct ahci_device {
pci_vendor_id_t         ad_vendor;
pci_product_id_t        ad_product;
  407         int                     (*ad_match)(struct pci_attach_args *);
  408         int                     (*ad_attach)(struct ahci_softc *,
  409                                     struct pci_attach_args *);
  410 };
  412 const struct ahci_device *ahci_lookup_device(struct pci_attach_args *);
  414 int                     ahci_no_match(struct pci_attach_args *);
  415 int                     ahci_vt8251_attach(struct ahci_softc *,
  416                             struct pci_attach_args *);
  418 static const struct ahci_device ahci_devices[] = {
  419         { PCI_VENDOR_VIATECH,   PCI_PRODUCT_VIATECH_VT8251_SATA,
ahci_no_match,      ahci_vt8251_attach }
  421 };
  423 int                     ahci_pci_match(struct device *, void *, void *);
  424 void                    ahci_pci_attach(struct device *, struct device *,
  425                             void *);
  427 struct cfattach ahci_pci_ca = {
  428         sizeof(struct ahci_softc), ahci_pci_match, ahci_pci_attach
  429 };
  431 struct cfattach ahci_jmb_ca = {
  432         sizeof(struct ahci_softc), ahci_pci_match, ahci_pci_attach
  433 };
  435 struct cfdriver ahci_cd = {
NULL, "ahci", DV_DULL
  437 };
  440 int                     ahci_map_regs(struct ahci_softc *,
  441                             struct pci_attach_args *);
  442 void                    ahci_unmap_regs(struct ahci_softc *,
  443                             struct pci_attach_args *);
  444 int                     ahci_map_intr(struct ahci_softc *,
  445                             struct pci_attach_args *, pci_intr_handle_t);
  446 #ifdef notyet
  447 void                    ahci_unmap_intr(struct ahci_softc *,
  448                             struct pci_attach_args *);
  449 #endif
  451 int                     ahci_init(struct ahci_softc *);
  452 int                     ahci_port_alloc(struct ahci_softc *, u_int);
  453 void                    ahci_port_free(struct ahci_softc *, u_int);
  455 int                     ahci_port_start(struct ahci_port *, int);
  456 int                     ahci_port_stop(struct ahci_port *, int);
  457 int                     ahci_port_clo(struct ahci_port *);
  458 int                     ahci_port_softreset(struct ahci_port *);
  459 int                     ahci_port_portreset(struct ahci_port *);
  461 int                     ahci_load_prdt(struct ahci_ccb *);
  462 void                    ahci_unload_prdt(struct ahci_ccb *);
  464 int                     ahci_poll(struct ahci_ccb *, int, void (*)(void *));
  465 void                    ahci_start(struct ahci_ccb *);
  467 void                    ahci_issue_pending_ncq_commands(struct ahci_port *);
  468 void                    ahci_issue_pending_commands(struct ahci_port *, int);
  470 int                     ahci_intr(void *);
u_int32_t               ahci_port_intr(struct ahci_port *, u_int32_t);
  473 struct ahci_ccb         *ahci_get_ccb(struct ahci_port *);
  474 void                    ahci_put_ccb(struct ahci_ccb *);
  476 struct ahci_ccb         *ahci_get_err_ccb(struct ahci_port *);
  477 void                    ahci_put_err_ccb(struct ahci_ccb *);
  479 int                     ahci_port_read_ncq_error(struct ahci_port *, int *);
  481 struct ahci_dmamem      *ahci_dmamem_alloc(struct ahci_softc *, size_t);
  482 void                    ahci_dmamem_free(struct ahci_softc *,
  483                             struct ahci_dmamem *);
u_int32_t               ahci_read(struct ahci_softc *, bus_size_t);
  486 void                    ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
  487 int                     ahci_wait_ne(struct ahci_softc *, bus_size_t,
u_int32_t, u_int32_t);
u_int32_t               ahci_pread(struct ahci_port *, bus_size_t);
  491 void                    ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
  492 int                     ahci_pwait_eq(struct ahci_port *, bus_size_t,
u_int32_t, u_int32_t);
  495 /* Wait for all bits in _b to be cleared */
  496 #define ahci_pwait_clr(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), 0)
  498 /* Wait for all bits in _b to be set */
  499 #define ahci_pwait_set(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), (_b))
  501 /* provide methods for atascsi to call */
  502 int                     ahci_ata_probe(void *, int);
  503 struct ata_xfer *       ahci_ata_get_xfer(void *, int);
  504 void                    ahci_ata_put_xfer(struct ata_xfer *);
  505 int                     ahci_ata_cmd(struct ata_xfer *);
  507 struct atascsi_methods ahci_atascsi_methods = {
ahci_ata_probe,
ahci_ata_get_xfer,
ahci_ata_cmd
  511 };
  513 /* ccb completions */
  514 void                    ahci_ata_cmd_done(struct ahci_ccb *);
  515 void                    ahci_ata_cmd_timeout(void *);
  516 void                    ahci_empty_done(struct ahci_ccb *);
  518 const struct ahci_device *
ahci_lookup_device(struct pci_attach_args *pa)
  520 {
  521         int                             i;
  522         const struct ahci_device        *ad;
  524         for (i = 0; i < (sizeof(ahci_devices) / sizeof(ahci_devices[0])); i++) {
ad = &ahci_devices[i];
  526                 if (ad->ad_vendor == PCI_VENDOR(pa->pa_id) &&
ad->ad_product == PCI_PRODUCT(pa->pa_id))
  528                         return (ad);
  529         }
  531         return (NULL);
  532 }
  534 int
ahci_no_match(struct pci_attach_args *pa)
  536 {
  537         return (0);
  538 }
  540 int
ahci_vt8251_attach(struct ahci_softc *sc, struct pci_attach_args *pa)
  542 {
sc->sc_flags |= AHCI_F_NO_NCQ;
  545         return (0);
  546 }
  548 int
ahci_pci_match(struct device *parent, void *match, void *aux)
  550 {
  551         struct pci_attach_args          *pa = aux;
  552         const struct ahci_device        *ad;
ad = ahci_lookup_device(pa);
  555         if (ad != NULL) {
  556                 /* the device may need special checks to see if it matches */
  557                 if (ad->ad_match != NULL)
  558                         return (ad->ad_match(pa));
  560                 return (2); /* match higher than pciide */
  561         }
  563         if (PCI_CLASS(pa->pa_class) == PCI_CLASS_MASS_STORAGE &&
PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_SATA &&
PCI_INTERFACE(pa->pa_class) == AHCI_PCI_INTERFACE)
  566                 return (2);
  568         return (0);
  569 }
  571 void
ahci_pci_attach(struct device *parent, struct device *self, void *aux)
  573 {
  574         struct ahci_softc               *sc = (struct ahci_softc *)self;
  575         struct pci_attach_args          *pa = aux;
  576         struct atascsi_attach_args      aaa;
  577         const struct ahci_device        *ad;
pci_intr_handle_t               ih;
u_int32_t                       cap, pi;
  580         int                             i;
ad = ahci_lookup_device(pa);
  583         if (ad != NULL && ad->ad_attach != NULL) {
  584                 if (ad->ad_attach(sc, pa) != 0) {
  585                         /* error should be printed by ad_attach */
  586                         return;
  587                 }
  588         }
  590         if (pci_intr_map(pa, &ih) != 0) {
printf(": unable to map interrupt\n");
  592                 return;
  593         }
printf(": %s,", pci_intr_string(pa->pa_pc, ih));
  596         if (ahci_map_regs(sc, pa) != 0) {
  597                 /* error already printed by ahci_map_regs */
  598                 return;
  599         }
  601         if (ahci_init(sc) != 0) {
  602                 /* error already printed by ahci_init */
  603                 goto unmap;
  604         }
  606         if (ahci_map_intr(sc, pa, ih) != 0) {
  607                 /* error already printed by ahci_map_intr */
  608                 goto unmap;
  609         }
printf("\n");
sc->sc_dmat = pa->pa_dmat;
cap = ahci_read(sc, AHCI_REG_CAP);
sc->sc_ncmds = AHCI_REG_CAP_NCS(cap);
  617 #ifdef AHCI_DEBUG
  618         if (ahcidebug & AHCI_D_VERBOSE) {
  619                 const char *gen;
  621                 switch (cap & AHCI_REG_CAP_ISS) {
  622                 case AHCI_REG_CAP_ISS_G1:
gen = "1 (1.5Gbps)";
  624                         break;
  625                 case AHCI_REG_CAP_ISS_G1_2:
gen = "1 (1.5Gbps) and 2 (3Gbps)";
  627                         break;
  628                 default:
gen = "unknown";
  630                         break;
  631                 }
printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
DEVNAME(sc), cap, AHCI_FMT_CAP,
AHCI_REG_CAP_NP(cap), sc->sc_ncmds, gen);
  636         }
  637 #endif
pi = ahci_read(sc, AHCI_REG_PI);
DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
DEVNAME(sc), pi);
  643 #ifdef AHCI_COALESCE
  644         /* Naive coalescing support - enable for all ports. */
  645         if (cap & AHCI_REG_CAP_CCCS) {
u_int16_t               ccc_timeout = 20;
u_int8_t                ccc_numcomplete = 12;
u_int32_t               ccc_ctl;
  650                 /* disable coalescing during reconfiguration. */
ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
ccc_ctl &= ~0x00000001;
ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
  656                 if (pi & sc->sc_ccc_mask) {
  657                         /* A conflict with the implemented port list? */
printf("%s: coalescing interrupt/implemented port list "
  659                             "conflict, PI: %08x, ccc_mask: %08x\n",
DEVNAME(sc), pi, sc->sc_ccc_mask);
sc->sc_ccc_mask = 0;
  662                         goto noccc;
  663                 }
  665                 /* ahci_port_start will enable each port when it starts. */
sc->sc_ccc_ports = pi;
sc->sc_ccc_ports_cur = 0;
  669                 /* program thresholds and enable overall coalescing. */
ccc_ctl &= ~0xffffff00;
ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
  675         }
noccc:
  677 #endif
  678         for (i = 0; i < AHCI_MAX_PORTS; i++) {
  679                 if (!ISSET(pi, 1 << i)) {
  680                         /* dont allocate stuff if the port isnt implemented */
  681                         continue;
  682                 }
  684                 if (ahci_port_alloc(sc, i) == ENOMEM)
  685                         goto freeports;
  686         }
bzero(&aaa, sizeof(aaa));
aaa.aaa_cookie = sc;
aaa.aaa_methods = &ahci_atascsi_methods;
aaa.aaa_minphys = minphys;
aaa.aaa_nports = AHCI_MAX_PORTS;
aaa.aaa_ncmds = sc->sc_ncmds;
aaa.aaa_capability = ASAA_CAP_NEEDS_RESERVED;
  695         if (!(sc->sc_flags & AHCI_F_NO_NCQ) && (cap & AHCI_REG_CAP_SNCQ))
aaa.aaa_capability |= ASAA_CAP_NCQ;
sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
  700         /* Enable interrupts */
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
  703         return;
freeports:
  706         for (i = 0; i < AHCI_MAX_PORTS; i++)
  707                 if (sc->sc_ports[i] != NULL)
ahci_port_free(sc, i);
unmap:
  710         /* Disable controller */
ahci_write(sc, AHCI_REG_GHC, 0);
ahci_unmap_regs(sc, pa);
  713         return;
  714 }
  716 int
ahci_map_regs(struct ahci_softc *sc, struct pci_attach_args *pa)
  718 {
pcireg_t                        maptype;
maptype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, AHCI_PCI_BAR);
  722         if (pci_mapreg_map(pa, AHCI_PCI_BAR, maptype, 0, &sc->sc_iot,
  723             &sc->sc_ioh, NULL, &sc->sc_ios, 0) != 0) {
printf(" unable to map registers\n");
  725                 return (1);
  726         }
  728         return (0);
  729 }
  731 void
ahci_unmap_regs(struct ahci_softc *sc, struct pci_attach_args *pa)
  733 {
bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
sc->sc_ios = 0;
  736 }
  738 int
ahci_map_intr(struct ahci_softc *sc, struct pci_attach_args *pa,
pci_intr_handle_t ih)
  741 {
sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_BIO,
ahci_intr, sc, DEVNAME(sc));
  744         if (sc->sc_ih == NULL) {
printf("%s: unable to map interrupt\n", DEVNAME(sc));
  746                 return (1);
  747         }
  749         return (0);
  750 }
  752 #ifdef notyet
  753 /* one day we'll have hotplug pci */
  754 void
ahci_unmap_intr(struct ahci_softc *sc, struct pci_attach_args *pa)
  756 {
pci_intr_disestablish(pa->pa_pc, sc->sc_ih);
  758 }
  759 #endif
  761 int
ahci_init(struct ahci_softc *sc)
  763 {
u_int32_t                       reg, cap, pi;
  765         const char                      *revision;
DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
AHCI_FMT_GHC);
  770         /* save BIOS initialised parameters, enable staggered spin up */
cap = ahci_read(sc, AHCI_REG_CAP);
cap &= AHCI_REG_CAP_SMPS;
cap |= AHCI_REG_CAP_SSS;
pi = ahci_read(sc, AHCI_REG_PI);
  776         if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
  777                 /* reset the controller */
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
  779                 if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
AHCI_REG_GHC_HR) != 0) {
printf(" unable to reset controller\n");
  782                         return (1);
  783                 }
  784         }
  786         /* enable ahci (global interrupts disabled) */
ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
  789         /* restore parameters */
ahci_write(sc, AHCI_REG_CAP, cap);
ahci_write(sc, AHCI_REG_PI, pi);
  793         /* check the revision */
reg = ahci_read(sc, AHCI_REG_VS);
  795         switch (reg) {
  796         case AHCI_REG_VS_0_95:
revision = "0.95";
  798                 break;
  799         case AHCI_REG_VS_1_0:
revision = "1.0";
  801                 break;
  802         case AHCI_REG_VS_1_1:
revision = "1.1";
  804                 break;
  806         default:
printf(" unsupported AHCI revision 0x%08x\n", reg);
  808                 return (1);
  809         }
printf(" AHCI %s", revision);
  813         return (0);
  814 }
  816 int
ahci_port_alloc(struct ahci_softc *sc, u_int port)
  818 {
  819         struct ahci_port                *ap;
  820         struct ahci_ccb                 *ccb;
u_int64_t                       dva;
u_int32_t                       cmd;
  823         struct ahci_cmd_hdr             *hdr;
  824         struct ahci_cmd_table           *table;
  825         int                             i, rc = ENOMEM;
ap = malloc(sizeof(struct ahci_port), M_DEVBUF, M_NOWAIT);
  828         if (ap == NULL) {
printf("%s: unable to allocate memory for port %d\n",
DEVNAME(sc), port);
  831                 goto reterr;
  832         }
bzero(ap, sizeof(struct ahci_port));
  835 #ifdef AHCI_DEBUG
snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
DEVNAME(sc), port);
  838 #endif
sc->sc_ports[port] = ap;
  841         if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
printf("%s: unable to create register window for port %d\n",
DEVNAME(sc), port);
  845                 goto freeport;
  846         }
ap->ap_sc = sc;
  849 #ifdef AHCI_COALESCE
ap->ap_num = port;
  851 #endif
TAILQ_INIT(&ap->ap_ccb_free);
TAILQ_INIT(&ap->ap_ccb_pending);
  855         /* Disable port interrupts */
ahci_pwrite(ap, AHCI_PREG_IE, 0);
  858         /* Sec 10.1.2 - deinitialise port if it is already running */
cmd = ahci_pread(ap, AHCI_PREG_CMD);
  860         if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
  863                 int r;
r = ahci_port_stop(ap, 1);
  866                 if (r) {
printf("%s: unable to disable %s, ignoring port %d\n",
DEVNAME(sc), r == 2 ? "CR" : "FR", port);
rc = ENXIO;
  870                         goto freeport;
  871                 }
  873                 /* Write DET to zero */
ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
  875         }
  877         /* Allocate RFIS */
ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
  879         if (ap->ap_dmamem_rfis == NULL)
  880                 goto nomem;
  882         /* Setup RFIS base address */
ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
  888         /* Enable FIS reception and activate port. */
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
  893         /* Check whether port activated.  Skip it if not. */
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
  895         if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
rc = ENXIO;
  897                 goto freeport;
  898         }
  900         /* Allocate a CCB for each command slot */
ap->ap_ccbs = malloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF,
M_NOWAIT);
  903         if (ap->ap_ccbs == NULL) {
printf("%s: unable to allocate command list for port %d\n",
DEVNAME(sc), port);
  906                 goto freeport;
  907         }
bzero(ap->ap_ccbs, sizeof(struct ahci_ccb) * sc->sc_ncmds);
  910         /* Command List Structures and Command Tables */
ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
sc->sc_ncmds * sizeof(struct ahci_cmd_table));
  915         if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
nomem:
printf("%s: unable to allocate DMA memory for port %d\n",
DEVNAME(sc), port);
  919                 goto freeport;
  920         }
  922         /* Setup command list base address */
dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
  927         /* Split CCB allocation into CCBs and assign to command header/table */
hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
  930         for (i = 0; i < sc->sc_ncmds; i++) {
ccb = &ap->ap_ccbs[i];
  933                 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
  934                     (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
  935                     &ccb->ccb_dmamap) != 0) {
printf("%s: unable to create dmamap for port %d "
  937                             "ccb %d\n", DEVNAME(sc), port, i);
  938                         goto freeport;
  939                 }
ccb->ccb_slot = i;
ccb->ccb_port = ap;
ccb->ccb_cmd_hdr = &hdr[i];
ccb->ccb_cmd_table = &table[i];
dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
ccb->ccb_slot * sizeof(struct ahci_cmd_table);
ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32));
ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva);
ccb->ccb_xa.fis =
  951                     (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
ccb->ccb_xa.tag = i;
ccb->ccb_xa.ata_put_xfer = ahci_ata_put_xfer;
ccb->ccb_xa.state = ATA_S_COMPLETE;
ahci_put_ccb(ccb);
  959         }
  961         /* Wait for ICC change to complete */
ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC);
  964         /* Reset port */
rc = ahci_port_portreset(ap);
  966         switch (rc) {
  967         case ENODEV:
  968                 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
  969                 case AHCI_PREG_SSTS_DET_DEV_NE:
printf("%s: device not communicating on port %d\n",
DEVNAME(sc), port);
  972                         break;
  973                 case AHCI_PREG_SSTS_DET_PHYOFFLINE:
printf("%s: PHY offline on port %d\n", DEVNAME(sc),
port);
  976                         break;
  977                 default:
DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
  979                             "on port %d\n", DEVNAME(sc), port);
  980                         break;
  981                 }
  982                 goto freeport;
  984         case EBUSY:
printf("%s: device on port %d didn't come ready, "
  986                     "TFD: 0x%b\n", DEVNAME(sc), port,
ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
  989                 /* Try a soft reset to clear busy */
rc = ahci_port_softreset(ap);
  991                 if (rc) {
printf("%s: unable to communicate "
  993                             "with device on port %d\n", DEVNAME(sc), port);
  994                         goto freeport;
  995                 }
  996                 break;
  998         default:
  999                 break;
 1000         }
DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
DEVNAME(sc), port);
 1004         /* Enable command transfers on port */
 1005         if (ahci_port_start(ap, 0)) {
printf("%s: failed to start command DMA on port %d, "
 1007                     "disabling\n", DEVNAME(sc), port);
rc = ENXIO;     /* couldn't start port */
 1009         }
 1011         /* Flush interrupts for port */
ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
ahci_write(sc, AHCI_REG_IS, 1 << port);
 1015         /* Enable port interrupts */
ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
AHCI_PREG_IE_UFE |
 1019 #ifdef AHCI_COALESCE
 1020             ((sc->sc_ccc_ports & (1 << port)) ? 0 : (AHCI_PREG_IE_SDBE |
AHCI_PREG_IE_DHRE))
 1022 #else
AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
 1024 #endif
 1025             );
freeport:
 1028         if (rc != 0)
ahci_port_free(sc, port);
reterr:
 1031         return (rc);
 1032 }
 1034 void
ahci_port_free(struct ahci_softc *sc, u_int port)
 1036 {
 1037         struct ahci_port                *ap = sc->sc_ports[port];
 1038         struct ahci_ccb                 *ccb;
 1040         /* Ensure port is disabled and its interrupts are flushed */
 1041         if (ap->ap_sc) {
ahci_pwrite(ap, AHCI_PREG_CMD, 0);
ahci_pwrite(ap, AHCI_PREG_IE, 0);
ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
ahci_write(sc, AHCI_REG_IS, 1 << port);
 1046         }
 1048         if (ap->ap_ccbs) {
 1049                 while ((ccb = ahci_get_ccb(ap)) != NULL)
bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
free(ap->ap_ccbs, M_DEVBUF);
 1052         }
 1054         if (ap->ap_dmamem_cmd_list)
ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
 1056         if (ap->ap_dmamem_rfis)
ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
 1058         if (ap->ap_dmamem_cmd_table)
ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
 1061         /* bus_space(9) says we dont free the subregions handle */
free(ap, M_DEVBUF);
sc->sc_ports[port] = NULL;
 1065 }
 1067 int
ahci_port_start(struct ahci_port *ap, int fre_only)
 1069 {
u_int32_t                       r;
 1072         /* Turn on FRE (and ST) */
r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
r |= AHCI_PREG_CMD_FRE;
 1075         if (!fre_only)
r |= AHCI_PREG_CMD_ST;
ahci_pwrite(ap, AHCI_PREG_CMD, r);
 1079 #ifdef AHCI_COALESCE
 1080         /* (Re-)enable coalescing on the port. */
 1081         if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
ap->ap_sc->sc_ccc_ports_cur);
 1085         }
 1086 #endif
 1088         /* Wait for FR to come on */
 1089         if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR))
 1090                 return (2);
 1092         /* Wait for CR to come on */
 1093         if (!fre_only && ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR))
 1094                 return (1);
 1096         return (0);
 1097 }
 1099 int
ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
 1101 {
u_int32_t                       r;
 1104 #ifdef AHCI_COALESCE
 1105         /* Disable coalescing on the port while it is stopped. */
 1106         if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
ap->ap_sc->sc_ccc_ports_cur);
 1110         }
 1111 #endif
 1113         /* Turn off ST (and FRE) */
r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
r &= ~AHCI_PREG_CMD_ST;
 1116         if (stop_fis_rx)
r &= ~AHCI_PREG_CMD_FRE;
ahci_pwrite(ap, AHCI_PREG_CMD, r);
 1120         /* Wait for CR to go off */
 1121         if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR))
 1122                 return (1);
 1124         /* Wait for FR to go off */
 1125         if (stop_fis_rx && ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR))
 1126                 return (2);
 1128         return (0);
 1129 }
 1131 /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
 1132 int
ahci_port_clo(struct ahci_port *ap)
 1134 {
 1135         struct ahci_softc               *sc = ap->ap_sc;
u_int32_t                       cmd;
 1138         /* Only attempt CLO if supported by controller */
 1139         if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
 1140                 return (1);
 1142         /* Issue CLO */
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
 1144 #ifdef DIAGNOSTIC
 1145         if (ISSET(cmd, AHCI_PREG_CMD_ST))
printf("%s: CLO requested while port running\n", PORTNAME(ap));
 1147 #endif
ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
 1150         /* Wait for completion */
 1151         if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) {
printf("%s: CLO did not complete\n", PORTNAME(ap));
 1153                 return (1);
 1154         }
 1156         return (0);
 1157 }
 1159 /* AHCI soft reset, Section 10.4.1 */
 1160 int
ahci_port_softreset(struct ahci_port *ap)
 1162 {
 1163         struct ahci_ccb                 *ccb = NULL;
 1164         struct ahci_cmd_hdr             *cmd_slot;
u_int8_t                        *fis;
 1166         int                             s, rc = EIO;
u_int32_t                       cmd;
DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
s = splbio();
 1173         /* Save previous command register state */
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
 1176         /* Idle port */
 1177         if (ahci_port_stop(ap, 0)) {
printf("%s: failed to stop port, cannot softreset\n",
PORTNAME(ap));
 1180                 goto err;
 1181         }
 1183         /* Request CLO if device appears hung */
 1184         if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
AHCI_PREG_TFD_STS_DRQ))
ahci_port_clo(ap);
 1188         /* Clear port errors to permit TFD transfer */
ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
 1191         /* Restart port */
 1192         if (ahci_port_start(ap, 0)) {
printf("%s: failed to start port, cannot softreset\n",
PORTNAME(ap));
 1195                 goto err;
 1196         }
 1198         /* Check whether CLO worked */
 1199         if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
 1203                     ? "failed" : "unsupported");
rc = EBUSY;
 1205                 goto err;
 1206         }
 1208         /* Prep first D2H command with SRST feature & clear busy/reset flags */
ccb = ahci_get_err_ccb(ap);
cmd_slot = ccb->ccb_cmd_hdr;
bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
fis = ccb->ccb_cmd_table->cfis;
fis[0] = 0x27;  /* Host to device */
fis[15] = 0x04; /* SRST DEVCTL */
cmd_slot->prdtl = 0;
cmd_slot->flags = htole16(5);   /* FIS length: 5 DWORDS */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); /* Write */
ccb->ccb_xa.state = ATA_S_PENDING;
 1224         if (ahci_poll(ccb, 1000, NULL) != 0)
 1225                 goto err;
 1227         /* Prep second D2H command to read status and complete reset sequence */
fis[0] = 0x27;  /* Host to device */
fis[15] = 0;
cmd_slot->prdtl = 0;
cmd_slot->flags = htole16(5);   /* FIS length: 5 DWORDS */
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
ccb->ccb_xa.state = ATA_S_PENDING;
 1236         if (ahci_poll(ccb, 1000, NULL) != 0)
 1237                 goto err;
 1239         if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) {
printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
AHCI_PFMT_TFD_STS);
rc = EBUSY;
 1245                 goto err;
 1246         }
rc = 0;
err:
 1250         if (ccb != NULL) {
 1251                 /* Abort our command, if it failed, by stopping command DMA. */
 1252                 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
printf("%s: stopping the port, softreset slot %d was "
 1254                             "still active.\n", PORTNAME(ap), ccb->ccb_slot);
ahci_port_stop(ap, 0);
 1256                 }
ccb->ccb_xa.state = ATA_S_ERROR;
ahci_put_err_ccb(ccb);
 1259         }
 1261         /* Restore saved CMD register state */
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
splx(s);
 1266         return (rc);
 1267 }
 1269 /* AHCI port reset, Section 10.4.2 */
 1270 int
ahci_port_portreset(struct ahci_port *ap)
 1272 {
u_int32_t                       cmd, r;
 1274         int                             rc;
DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
 1278         /* Save previous command register state */
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
 1281         /* Clear ST, ignoring failure */
ahci_port_stop(ap, 0);
 1284         /* Perform device detection */
ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
delay(10000);
r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_SPD_ANY |
AHCI_PREG_SCTL_DET_INIT;
ahci_pwrite(ap, AHCI_PREG_SCTL, r);
delay(10000);   /* wait at least 1ms for COMRESET to be sent */
r &= ~AHCI_PREG_SCTL_DET_INIT;
r |= AHCI_PREG_SCTL_DET_NONE;
ahci_pwrite(ap, AHCI_PREG_SCTL, r);
delay(10000);
 1296         /* Wait for device to be detected and communications established */
 1297         if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET,
AHCI_PREG_SSTS_DET_DEV)) {
rc = ENODEV;
 1300                 goto err;
 1301         }
 1303         /* Clear SERR (incl X bit), so TFD can update */
ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
 1306         /* Wait for device to become ready */
 1307         /* XXX maybe more than the default wait is appropriate here? */
 1308         if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) {
rc = EBUSY;
 1311                 goto err;
 1312         }
rc = 0;
err:
 1316         /* Restore preserved port state */
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
 1319         return (rc);
 1320 }
 1322 int
ahci_load_prdt(struct ahci_ccb *ccb)
 1324 {
 1325         struct ahci_port                *ap = ccb->ccb_port;
 1326         struct ahci_softc               *sc = ap->ap_sc;
 1327         struct ata_xfer                 *xa = &ccb->ccb_xa;
 1328         struct ahci_prdt                *prdt = ccb->ccb_cmd_table->prdt, *prd;
bus_dmamap_t                    dmap = ccb->ccb_dmamap;
 1330         struct ahci_cmd_hdr             *cmd_slot = ccb->ccb_cmd_hdr;
u_int64_t                       addr;
 1332         int                             i, error;
 1334         if (xa->datalen == 0) {
ccb->ccb_cmd_hdr->prdtl = 0;
 1336                 return (0);
 1337         }
error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
 1340             (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
 1341         if (error != 0) {
printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
 1343                 return (1);
 1344         }
 1346         for (i = 0; i < dmap->dm_nsegs; i++) {
prd = &prdt[i];
addr = dmap->dm_segs[i].ds_addr;
prd->dba_hi = htole32((u_int32_t)(addr >> 32));
prd->dba_lo = htole32((u_int32_t)addr);
 1352 #ifdef DIAGNOSTIC
 1353                 if (addr & 1) {
printf("%s: requested DMA at an odd address %llx\n",
PORTNAME(ap), (unsigned long long)addr);
 1356                         return (1);
 1357                 }
 1358                 if (dmap->dm_segs[i].ds_len & 1) {
printf("%s: requested DMA length %d is not even\n",
PORTNAME(ap), (int)dmap->dm_segs[i].ds_len);
 1361                         return (1);
 1362                 }
 1363 #endif
prd->flags = htole32(dmap->dm_segs[i].ds_len - 1);
 1365         }
 1366         if (xa->flags & ATA_F_PIO)
prd->flags |= htole32(AHCI_PRDT_FLAG_INTR);
cmd_slot->prdtl = htole16(dmap->dm_nsegs);
bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
 1372             (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
BUS_DMASYNC_PREWRITE);
 1375         return (0);
 1376 }
 1378 void
ahci_unload_prdt(struct ahci_ccb *ccb)
 1380 {
 1381         struct ahci_port                *ap = ccb->ccb_port;
 1382         struct ahci_softc               *sc = ap->ap_sc;
 1383         struct ata_xfer                 *xa = &ccb->ccb_xa;
bus_dmamap_t                    dmap = ccb->ccb_dmamap;
 1386         if (xa->datalen != 0) {
bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
 1388                     (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, dmap);
 1393                 if (ccb->ccb_xa.flags & ATA_F_NCQ)
xa->resid = 0;
 1395                 else
xa->resid = xa->datalen -
letoh32(ccb->ccb_cmd_hdr->prdbc);
 1398         }
 1399 }
 1401 int
ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
 1403 {
 1404         struct ahci_port                *ap = ccb->ccb_port;
 1405         int                             s;
s = splbio();
ahci_start(ccb);
 1409         do {
 1410                 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
 1411                     1 << ccb->ccb_slot)) {
splx(s);
 1413                         return (0);
 1414                 }
delay(1000);
 1417         } while (--timeout > 0);
 1419         /* Run timeout while at splbio, otherwise ahci_intr could interfere. */
 1420         if (timeout_fn != NULL)
timeout_fn(ccb);
splx(s);
 1425         return (1);
 1426 }
 1428 void
ahci_start(struct ahci_ccb *ccb)
 1430 {
 1431         struct ahci_port                *ap = ccb->ccb_port;
 1432         struct ahci_softc               *sc = ap->ap_sc;
KASSERT(ccb->ccb_xa.state == ATA_S_PENDING);
 1436         /* Zero transferred byte count before transfer */
ccb->ccb_cmd_hdr->prdbc = 0;
 1439         /* Sync command list entry and corresponding command table entry */
bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
 1442             sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
ccb->ccb_slot * sizeof(struct ahci_cmd_table),
 1445             sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
 1447         /* Prepare RFIS area for write by controller */
bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
 1449             sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
 1451         if (ccb->ccb_xa.flags & ATA_F_NCQ) {
 1452                 /* Issue NCQ commands only when there are no outstanding
 1453                  * standard commands. */
 1454                 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending))
TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
 1456                 else {
KASSERT(ap->ap_active_cnt == 0);
ap->ap_sactive |= (1 << ccb->ccb_slot);
ccb->ccb_xa.state = ATA_S_ONCHIP;
ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
 1462                 }
 1463         } else {
 1464                 /* Wait for all NCQ commands to finish before issuing standard
 1465                  * command. */
 1466                 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
 1468                 else if (ap->ap_active_cnt < 2) {
ap->ap_active |= 1 << ccb->ccb_slot;
ccb->ccb_xa.state = ATA_S_ONCHIP;
ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
ap->ap_active_cnt++;
 1473                 }
 1474         }
 1475 }
 1477 void
ahci_issue_pending_ncq_commands(struct ahci_port *ap)
 1479 {
 1480         struct ahci_ccb                 *nextccb;
u_int32_t                       sact_change = 0;
KASSERT(ap->ap_active_cnt == 0);
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
 1486         if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
 1487                 return;
 1489         /* Start all the NCQ commands at the head of the pending list. */
 1490         do {
TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
sact_change |= 1 << nextccb->ccb_slot;
nextccb->ccb_xa.state = ATA_S_ONCHIP;
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
 1495         } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ));
ap->ap_sactive |= sact_change;
ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
 1501         return;
 1502 }
 1504 void
ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
 1506 {
 1507         struct ahci_ccb                 *nextccb;
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
 1510         if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
KASSERT(last_was_ncq == 0);     /* otherwise it should have
 1512                                                  * been started already. */
 1514                 /* Issue NCQ commands only when there are no outstanding
 1515                  * standard commands. */
ap->ap_active_cnt--;
 1517                 if (ap->ap_active == 0)
ahci_issue_pending_ncq_commands(ap);
 1519                 else
KASSERT(ap->ap_active_cnt == 1);
 1521         } else if (nextccb) {
 1522                 if (ap->ap_sactive != 0 || last_was_ncq)
KASSERT(ap->ap_active_cnt == 0);
 1525                 /* Wait for all NCQ commands to finish before issuing standard
 1526                  * command. */
 1527                 if (ap->ap_sactive != 0)
 1528                         return;
 1530                 /* Keep up to 2 standard commands on-chip at a time. */
 1531                 do {
TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
ap->ap_active |= 1 << nextccb->ccb_slot;
nextccb->ccb_xa.state = ATA_S_ONCHIP;
ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
 1536                         if (last_was_ncq)
ap->ap_active_cnt++;
 1538                         if (ap->ap_active_cnt == 2)
 1539                                 break;
KASSERT(ap->ap_active_cnt == 1);
nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
 1542                 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
 1543         } else if (!last_was_ncq) {
KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
 1546                 /* Standard command finished, none waiting to start. */
ap->ap_active_cnt--;
 1548         } else {
KASSERT(ap->ap_active_cnt == 0);
 1551                 /* NCQ command finished. */
 1552         }
 1553 }
 1555 int
ahci_intr(void *arg)
 1557 {
 1558         struct ahci_softc               *sc = arg;
u_int32_t                       is, ack = 0;
 1560         int                             port;
 1562         /* Read global interrupt status */
is = ahci_read(sc, AHCI_REG_IS);
 1564         if (is == 0)
 1565                 return (0);
ack = is;
 1568 #ifdef AHCI_COALESCE
 1569         /* Check coalescing interrupt first */
 1570         if (is & sc->sc_ccc_mask) {
DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
DEVNAME(sc));
is &= ~sc->sc_ccc_mask;
is |= sc->sc_ccc_ports_cur;
 1575         }
 1576 #endif
 1578         /* Process interrupts for each port */
 1579         while (is) {
port = ffs(is) - 1;
 1581                 if (sc->sc_ports[port])
ahci_port_intr(sc->sc_ports[port],
AHCI_PREG_CI_ALL_SLOTS);
is &= ~(1 << port);
 1585         }
 1587         /* Finally, acknowledge global interrupt */
ahci_write(sc, AHCI_REG_IS, ack);
 1590         return (1);
 1591 }
u_int32_t
ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
 1595 {
 1596         struct ahci_softc               *sc = ap->ap_sc;
u_int32_t                       is, ci_saved, ci_masked, processed = 0;
 1598         int                             slot, need_restart = 0;
 1599         struct ahci_ccb                 *ccb;
 1600         volatile u_int32_t              *active;
 1601 #ifdef DIAGNOSTIC
u_int32_t                       tmp;
 1603 #endif
is = ahci_pread(ap, AHCI_PREG_IS);
 1607         /* Ack port interrupt only if checking all command slots. */
 1608         if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
ahci_pwrite(ap, AHCI_PREG_IS, is);
 1611         if (is)
DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
is, AHCI_PFMT_IS);
 1615         if (ap->ap_sactive) {
 1616                 /* Active NCQ commands - use SActive instead of CI */
KASSERT(ap->ap_active == 0);
KASSERT(ap->ap_active_cnt == 0);
ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
active = &ap->ap_sactive;
 1621         } else {
 1622                 /* Save CI */
ci_saved = ahci_pread(ap, AHCI_PREG_CI);
active = &ap->ap_active;
 1625         }
 1627         /* Command failed.  See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
 1628         if (is & AHCI_PREG_IS_TFES) {
u_int32_t               tfd, serr;
 1630                 int                     err_slot;
tfd = ahci_pread(ap, AHCI_PREG_TFD);
serr = ahci_pread(ap, AHCI_PREG_SERR);
 1635                 if (ap->ap_sactive == 0) {
 1636                         /* Errored slot is easy to determine from CMD. */
err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
AHCI_PREG_CMD));
ccb = &ap->ap_ccbs[err_slot];
 1641                         /* Preserve received taskfile data from the RFIS. */
memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
 1643                             sizeof(struct ata_fis_d2h));
 1644                 } else
err_slot = -1;  /* Must extract error from log page */
DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
 1648                     " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
AHCI_PFMT_SERR_DIAG);
 1653                 /* Turn off ST to clear CI and SACT. */
ahci_port_stop(ap, 0);
need_restart = 1;
 1657                 /* Clear SERR to enable capturing new errors. */
ahci_pwrite(ap, AHCI_PREG_SERR, serr);
 1660                 /* Acknowledge the interrupts we can recover from. */
ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
AHCI_PREG_IS_IFS);
is = ahci_pread(ap, AHCI_PREG_IS);
 1665                 /* If device hasn't cleared its busy status, try to idle it. */
 1666                 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
printf("%s: attempting to idle device\n", PORTNAME(ap));
 1668                         if (ahci_port_softreset(ap)) {
printf("%s: failed to soft reset device\n",
PORTNAME(ap));
 1671                                 if (ahci_port_portreset(ap)) {
printf("%s: failed to port reset "
 1673                                             "device, give up on it\n",
PORTNAME(ap));
 1675                                         goto fatal;
 1676                                 }
 1677                         }
 1679                         /* Had to reset device, can't gather extended info. */
 1680                 } else if (ap->ap_sactive) {
 1681                         /* Recover the NCQ error from log page 10h. */
ahci_port_read_ncq_error(ap, &err_slot);
 1683                         if (err_slot < 0)
 1684                                 goto failall;
DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
PORTNAME(ap), err_slot);
ccb = &ap->ap_ccbs[err_slot];
 1690                 } else {
 1691                         /* Didn't reset, could gather extended info from log. */
 1692                 }
 1694                 /*
 1695                  * If we couldn't determine the errored slot, reset the port
 1696                  * and fail all the active slots.
 1697                  */
 1698                 if (err_slot == -1) {
 1699                         if (ahci_port_softreset(ap) != 0 &&
ahci_port_portreset(ap) != 0) {
printf("%s: couldn't reset after NCQ error, "
 1702                                     "disabling device.\n", PORTNAME(ap));
 1703                                 goto fatal;
 1704                         }
printf("%s: couldn't recover NCQ error, failing "
 1706                             "all outstanding commands.\n", PORTNAME(ap));
 1707                         goto failall;
 1708                 }
 1710                 /* Clear the failed command in saved CI so completion runs. */
ci_saved &= ~(1 << err_slot);
 1713                 /* Note the error in the ata_xfer. */
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
ccb->ccb_xa.state = ATA_S_ERROR;
 1717 #ifdef DIAGNOSTIC
 1718                 /* There may only be one outstanding standard command now. */
 1719                 if (ap->ap_sactive == 0) {
tmp = ci_saved;
 1721                         if (tmp) {
slot = ffs(tmp) - 1;
tmp &= ~(1 << slot);
KASSERT(tmp == 0);
 1725                         }
 1726                 }
 1727 #endif
 1728         }
 1730         /* Check for remaining errors - they are fatal. */
 1731         if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
PORTNAME(ap), is, AHCI_PFMT_IS);
 1736                 /* XXX try recovery first */
 1737                 goto fatal;
 1738         }
 1740         /* Fail all outstanding commands if we know the port won't recover. */
 1741         if (ap->ap_state == AP_S_FATAL_ERROR) {
fatal:
ap->ap_state = AP_S_FATAL_ERROR;
failall:
 1746                 /* Ensure port is shut down. */
ahci_port_stop(ap, 1);
 1749                 /* Error all the active slots. */
ci_masked = ci_saved & *active;
 1751                 while (ci_masked) {
slot = ffs(ci_masked) - 1;
ccb = &ap->ap_ccbs[slot];
ci_masked &= ~(1 << slot);
ccb->ccb_xa.state = ATA_S_ERROR;
 1756                 }
 1758                 /* Run completion for all active slots. */
ci_saved &= ~*active;
 1761                 /* Don't restart the port if our problems were deemed fatal. */
 1762                 if (ap->ap_state == AP_S_FATAL_ERROR)
need_restart = 0;
 1764         }
 1766         /*
 1767          * CCB completion is detected by noticing its slot's bit in CI has
 1768          * changed to zero some time after we activated it.
 1769          * If we are polling, we may only be interested in particular slot(s).
 1770          */
ci_masked = ~ci_saved & *active & ci_mask;
 1772         while (ci_masked) {
slot = ffs(ci_masked) - 1;
ccb = &ap->ap_ccbs[slot];
ci_masked &= ~(1 << slot);
DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
 1779                     " (error)" : "");
bus_dmamap_sync(sc->sc_dmat,
AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
 1784                     sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
bus_dmamap_sync(sc->sc_dmat,
AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
ccb->ccb_slot * sizeof(struct ahci_cmd_table),
 1789                     sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
bus_dmamap_sync(sc->sc_dmat,
AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
 1793                     sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
 1795                 *active &= ~(1 << ccb->ccb_slot);
ccb->ccb_done(ccb);
processed |= 1 << ccb->ccb_slot;
 1799         }
 1801         if (need_restart) {
 1802                 /* Restart command DMA on the port */
ahci_port_start(ap, 0);
 1805                 /* Re-enable outstanding commands on port. */
 1806                 if (ci_saved) {
 1807 #ifdef DIAGNOSTIC
tmp = ci_saved;
 1809                         while (tmp) {
slot = ffs(tmp) - 1;
tmp &= ~(1 << slot);
ccb = &ap->ap_ccbs[slot];
KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
 1815                                     (!!ap->ap_sactive));
 1816                         }
 1817 #endif
DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
 1819                             "re-enabling%s slots %08x\n", PORTNAME(ap),
ap->ap_sactive ? " NCQ" : "", ci_saved);
 1822                         if (ap->ap_sactive)
ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
 1825                 }
 1826         }
 1828         return (processed);
 1829 }
 1831 struct ahci_ccb *
ahci_get_ccb(struct ahci_port *ap)
 1833 {
 1834         struct ahci_ccb                 *ccb;
ccb = TAILQ_FIRST(&ap->ap_ccb_free);
 1837         if (ccb != NULL) {
KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
ccb->ccb_xa.state = ATA_S_SETUP;
 1841         }
 1843         return (ccb);
 1844 }
 1846 void
ahci_put_ccb(struct ahci_ccb *ccb)
 1848 {
 1849         struct ahci_port                *ap = ccb->ccb_port;
 1851 #ifdef DIAGNOSTIC
 1852         if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
ccb->ccb_xa.state != ATA_S_TIMEOUT &&
ccb->ccb_xa.state != ATA_S_ERROR) {
printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
 1856                     "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
ccb->ccb_slot);
 1858         }
 1859 #endif
ccb->ccb_xa.state = ATA_S_PUT;
TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
 1863 }
 1865 struct ahci_ccb *
ahci_get_err_ccb(struct ahci_port *ap)
 1867 {
 1868         struct ahci_ccb *err_ccb;
u_int32_t sact;
splassert(IPL_BIO);
 1873         /* No commands may be active on the chip. */
sact = ahci_pread(ap, AHCI_PREG_SACT);
 1875         if (sact != 0)
printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
 1879 #ifdef DIAGNOSTIC
KASSERT(ap->ap_err_busy == 0);
ap->ap_err_busy = 1;
 1882 #endif
 1883         /* Save outstanding command state. */
ap->ap_err_saved_active = ap->ap_active;
ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
ap->ap_err_saved_sactive = ap->ap_sactive;
 1888         /*
 1889          * Pretend we have no commands outstanding, so that completions won't
 1890          * run prematurely.
 1891          */
ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
 1894         /*
 1895          * Grab a CCB to use for error recovery.  This should never fail, as
 1896          * we ask atascsi to reserve one for us at init time.
 1897          */
err_ccb = ahci_get_ccb(ap);
KASSERT(err_ccb != NULL);
err_ccb->ccb_xa.flags = 0;
err_ccb->ccb_done = ahci_empty_done;
 1903         return err_ccb;
 1904 }
 1906 void
ahci_put_err_ccb(struct ahci_ccb *ccb)
 1908 {
 1909         struct ahci_port *ap = ccb->ccb_port;
u_int32_t sact;
splassert(IPL_BIO);
 1914 #ifdef DIAGNOSTIC
KASSERT(ap->ap_err_busy);
 1916 #endif
 1917         /* No commands may be active on the chip */
sact = ahci_pread(ap, AHCI_PREG_SACT);
 1919         if (sact != 0)
printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
 1923         /* Done with the CCB */
ahci_put_ccb(ccb);
 1926         /* Restore outstanding command state */
ap->ap_sactive = ap->ap_err_saved_sactive;
ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
ap->ap_active = ap->ap_err_saved_active;
 1931 #ifdef DIAGNOSTIC
ap->ap_err_busy = 0;
 1933 #endif
 1934 }
 1936 int
ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp)
 1938 {
 1939         struct ahci_ccb                 *ccb;
 1940         struct ahci_cmd_hdr             *cmd_slot;
u_int32_t                       cmd;
 1942         struct ata_fis_h2d              *fis;
 1943         int                             rc = EIO;
DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
 1947         /* Save command register state. */
cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
 1950         /* Port should have been idled already.  Start it. */
KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
ahci_port_start(ap, 0);
 1954         /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
ccb = ahci_get_err_ccb(ap);
ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
ccb->ccb_xa.data = ap->ap_err_scratch;
ccb->ccb_xa.datalen = 512;
cmd_slot = ccb->ccb_cmd_hdr;
bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
fis->type = ATA_FIS_TYPE_H2D;
fis->flags = ATA_H2D_FLAGS_CMD;
fis->command = ATA_C_READ_LOG_EXT;
fis->lba_low = 0x10;            /* queued error log page (10h) */
fis->sector_count = 1;          /* number of sectors (1) */
fis->sector_count_exp = 0;
fis->lba_mid = 0;               /* starting offset */
fis->lba_mid_exp = 0;
fis->device = 0;
cmd_slot->flags = htole16(5);   /* FIS length: 5 DWORDS */
 1975         if (ahci_load_prdt(ccb) != 0) {
rc = ENOMEM;    /* XXX caller must abort all commands */
 1977                 goto err;
 1978         }
ccb->ccb_xa.state = ATA_S_PENDING;
 1981         if (ahci_poll(ccb, 1000, NULL) != 0)
 1982                 goto err;
rc = 0;
err:
 1986         /* Abort our command, if it failed, by stopping command DMA. */
 1987         if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
printf("%s: log page read failed, slot %d was still active.\n",
PORTNAME(ap), ccb->ccb_slot);
ahci_port_stop(ap, 0);
 1991         }
 1993         /* Done with the error CCB now. */
ahci_unload_prdt(ccb);
ahci_put_err_ccb(ccb);
 1997         /* Extract failed register set and tags from the scratch space. */
 1998         if (rc == 0) {
 1999                 struct ata_log_page_10h         *log;
 2000                 int                             err_slot;
log = (struct ata_log_page_10h *)ap->ap_err_scratch;
 2003                 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
 2004                         /* Not queued bit was set - wasn't an NCQ error? */
printf("%s: read NCQ error page, but not an NCQ "
 2006                             "error?\n", PORTNAME(ap));
rc = ESRCH;
 2008                 } else {
 2009                         /* Copy back the log record as a D2H register FIS. */
 2010                         *err_slotp = err_slot = log->err_regs.type &
ATA_LOG_10H_TYPE_TAG_MASK;
ccb = &ap->ap_ccbs[err_slot];
memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
 2015                             sizeof(struct ata_fis_d2h));
ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
ccb->ccb_xa.rfis.flags = 0;
 2018                 }
 2019         }
 2021         /* Restore saved CMD register state */
ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
 2024         return (rc);
 2025 }
 2027 struct ahci_dmamem *
ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
 2029 {
 2030         struct ahci_dmamem              *adm;
 2031         int                             nsegs;
adm = malloc(sizeof(struct ahci_dmamem), M_DEVBUF, M_NOWAIT);
 2034         if (adm == NULL)
 2035                 return (NULL);
bzero(adm, sizeof(struct ahci_dmamem));
adm->adm_size = size;
 2040         if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
 2042                 goto admfree;
 2044         if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
 2045             1, &nsegs, BUS_DMA_NOWAIT) != 0)
 2046                 goto destroy;
 2048         if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
 2049             &adm->adm_kva, BUS_DMA_NOWAIT) != 0)
 2050                 goto free;
 2052         if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
NULL, BUS_DMA_NOWAIT) != 0)
 2054                 goto unmap;
bzero(adm->adm_kva, size);
 2058         return (adm);
unmap:
bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
free:
bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
destroy:
bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
admfree:
free(adm, M_DEVBUF);
 2069         return (NULL);
 2070 }
 2072 void
ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
 2074 {
bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
free(adm, M_DEVBUF);
 2080 }
u_int32_t
ahci_read(struct ahci_softc *sc, bus_size_t r)
 2084 {
bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
BUS_SPACE_BARRIER_READ);
 2087         return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
 2088 }
 2090 void
ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
 2092 {
bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
BUS_SPACE_BARRIER_WRITE);
 2096 }
 2098 int
ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
u_int32_t target)
 2101 {
 2102         int                             i;
 2104         for (i = 0; i < 1000; i++) {
 2105                 if ((ahci_read(sc, r) & mask) != target)
 2106                         return (0);
delay(1000);
 2108         }
 2110         return (1);
 2111 }
u_int32_t
ahci_pread(struct ahci_port *ap, bus_size_t r)
 2115 {
bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
BUS_SPACE_BARRIER_READ);
 2118         return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
 2119 }
 2121 void
ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
 2123 {
bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
BUS_SPACE_BARRIER_WRITE);
 2127 }
 2129 int
ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
u_int32_t target)
 2132 {
 2133         int                             i;
 2135         for (i = 0; i < 1000; i++) {
 2136                 if ((ahci_pread(ap, r) & mask) == target)
 2137                         return (0);
delay(1000);
 2139         }
 2141         return (1);
 2142 }
 2144 int
ahci_ata_probe(void *xsc, int port)
 2146 {
 2147         struct ahci_softc               *sc = xsc;
 2148         struct ahci_port                *ap = sc->sc_ports[port];
u_int32_t                       sig;
 2151         if (ap == NULL)
 2152                 return (ATA_PORT_T_NONE);
sig = ahci_pread(ap, AHCI_PREG_SIG);
 2155         if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
 2156                 return (ATA_PORT_T_ATAPI);
 2157         else
 2158                 return (ATA_PORT_T_DISK);
 2159 }
 2161 struct ata_xfer *
ahci_ata_get_xfer(void *aaa_cookie, int port)
 2163 {
 2164         struct ahci_softc               *sc = aaa_cookie;
 2165         struct ahci_port                *ap = sc->sc_ports[port];
 2166         struct ahci_ccb                 *ccb;
splassert(IPL_BIO);
ccb = ahci_get_ccb(ap);
 2171         if (ccb == NULL) {
DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
PORTNAME(ap));
 2174                 return (NULL);
 2175         }
DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
PORTNAME(ap), ccb->ccb_slot);
 2180         return ((struct ata_xfer *)ccb);
 2181 }
 2183 void
ahci_ata_put_xfer(struct ata_xfer *xa)
 2185 {
 2186         struct ahci_ccb                 *ccb = (struct ahci_ccb *)xa;
splassert(IPL_BIO);
DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
ahci_put_ccb(ccb);
 2193 }
 2195 int
ahci_ata_cmd(struct ata_xfer *xa)
 2197 {
 2198         struct ahci_ccb                 *ccb = (struct ahci_ccb *)xa;
 2199         struct ahci_cmd_hdr             *cmd_slot;
 2200         int                             s;
KASSERT(xa->state == ATA_S_SETUP);
 2204         if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
 2205                 goto failcmd;
ccb->ccb_done = ahci_ata_cmd_done;
cmd_slot = ccb->ccb_cmd_hdr;
cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */
 2212         if (xa->flags & ATA_F_WRITE)
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
 2215         if (xa->flags & ATA_F_PACKET)
cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A);
 2218         if (ahci_load_prdt(ccb) != 0)
 2219                 goto failcmd;
timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
xa->state = ATA_S_PENDING;
 2225         if (xa->flags & ATA_F_POLL) {
ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
 2227                 return (ATA_COMPLETE);
 2228         }
timeout_add(&xa->stimeout, (xa->timeout * hz) / 1000);
s = splbio();
ahci_start(ccb);
splx(s);
 2235         return (ATA_QUEUED);
failcmd:
s = splbio();
xa->state = ATA_S_ERROR;
xa->complete(xa);
splx(s);
 2242         return (ATA_ERROR);
 2243 }
 2245 void
ahci_ata_cmd_done(struct ahci_ccb *ccb)
 2247 {
 2248         struct ata_xfer                 *xa = &ccb->ccb_xa;
timeout_del(&xa->stimeout);
 2252         if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
ahci_issue_pending_commands(ccb->ccb_port,
xa->flags & ATA_F_NCQ);
ahci_unload_prdt(ccb);
 2258         if (xa->state == ATA_S_ONCHIP)
xa->state = ATA_S_COMPLETE;
 2260 #ifdef DIAGNOSTIC
 2261         else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
 2263                     "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
ccb->ccb_slot);
 2265 #endif
 2266         if (xa->state != ATA_S_TIMEOUT)
xa->complete(xa);
 2268 }
 2270 void
ahci_ata_cmd_timeout(void *arg)
 2272 {
 2273         struct ahci_ccb                 *ccb = arg;
 2274         struct ata_xfer                 *xa = &ccb->ccb_xa;
 2275         struct ahci_port                *ap = ccb->ccb_port;
 2276         int                             s, ccb_was_started, ncq_cmd;
 2277         volatile u_int32_t              *active;
s = splbio();
ncq_cmd = (xa->flags & ATA_F_NCQ);
active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
 2284         if (ccb->ccb_xa.state == ATA_S_PENDING) {
DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
 2286                     "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
ccb_was_started = 0;
 2289         } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
 2290             1 << ccb->ccb_slot)) {
DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
 2292                     "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
 2293                 goto ret;
 2294         } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
 2296                     "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
ISSET(*active, 1 << ccb->ccb_slot) ?
 2298                     " but slot is still active?" : ".");
 2299                 goto ret;
 2300         } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
 2302             1 << ccb->ccb_slot)) {
DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
 2304                     "IRQ handler didn't detect it.  Why?\n", PORTNAME(ap),
ccb->ccb_slot);
 2306                 *active &= ~(1 << ccb->ccb_slot);
ccb->ccb_done(ccb);
 2308                 goto ret;
 2309         } else {
ccb_was_started = 1;
 2311         }
 2313         /* Complete the slot with a timeout error. */
ccb->ccb_xa.state = ATA_S_TIMEOUT;
 2315         *active &= ~(1 << ccb->ccb_slot);
DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
ccb->ccb_done(ccb);     /* This won't issue pending commands or run the
 2318                                    atascsi completion. */
 2320         /* Reset port to abort running command. */
 2321         if (ccb_was_started) {
DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
 2323                     "in slot %d, active %08x\n", PORTNAME(ap), ncq_cmd ? " NCQ"
 2324                     : "", ccb->ccb_slot, *active);
 2325                 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap)
 2326                     != 0) {
printf("%s: failed to reset port during timeout "
 2328                             "handling, disabling it\n", PORTNAME(ap));
ap->ap_state = AP_S_FATAL_ERROR;
 2330                 }
 2332                 /* Restart any other commands that were aborted by the reset. */
 2333                 if (*active) {
DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
 2335                             "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
 2336                             *active);
 2337                         if (ncq_cmd)
ahci_pwrite(ap, AHCI_PREG_SACT, *active);
ahci_pwrite(ap, AHCI_PREG_CI, *active);
 2340                 }
 2341         }
 2343         /* Issue any pending commands now. */
DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
 2345         if (ccb_was_started)
ahci_issue_pending_commands(ap, ncq_cmd);
 2347         else if (ap->ap_active == 0)
ahci_issue_pending_ncq_commands(ap);
 2350         /* Complete the timed out ata_xfer I/O (may generate new I/O). */
DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
xa->complete(xa);
DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
ret:
splx(s);
 2357 }
 2359 void
ahci_empty_done(struct ahci_ccb *ccb)
 2361 {
ccb->ccb_xa.state = ATA_S_COMPLETE;
 2363 }