Showing error 1839

User: Jiri Slaby
Error type: Invalid Pointer Dereference
Error type description: A pointer which is invalid is being dereferenced
File location: drivers/scsi/aic7xxx/aic79xx_core.c
Line in file: 9690
Project: Linux Kernel
Project version: 2.6.28
Tools: Smatch (1.59)
Entered: 2013-09-11 08:47:26 UTC


Source:

    1/*
    2 * Core routines and tables shareable across OS platforms.
    3 *
    4 * Copyright (c) 1994-2002 Justin T. Gibbs.
    5 * Copyright (c) 2000-2003 Adaptec Inc.
    6 * All rights reserved.
    7 *
    8 * Redistribution and use in source and binary forms, with or without
    9 * modification, are permitted provided that the following conditions
   10 * are met:
   11 * 1. Redistributions of source code must retain the above copyright
   12 *    notice, this list of conditions, and the following disclaimer,
   13 *    without modification.
   14 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
   15 *    substantially similar to the "NO WARRANTY" disclaimer below
   16 *    ("Disclaimer") and any redistribution must be conditioned upon
   17 *    including a substantially similar Disclaimer requirement for further
   18 *    binary redistribution.
   19 * 3. Neither the names of the above-listed copyright holders nor the names
   20 *    of any contributors may be used to endorse or promote products derived
   21 *    from this software without specific prior written permission.
   22 *
   23 * Alternatively, this software may be distributed under the terms of the
   24 * GNU General Public License ("GPL") version 2 as published by the Free
   25 * Software Foundation.
   26 *
   27 * NO WARRANTY
   28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
   31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
   32 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
   36 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
   37 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   38 * POSSIBILITY OF SUCH DAMAGES.
   39 *
   40 * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#250 $
   41 */
   42
   43#ifdef __linux__
   44#include "aic79xx_osm.h"
   45#include "aic79xx_inline.h"
   46#include "aicasm/aicasm_insformat.h"
   47#else
   48#include <dev/aic7xxx/aic79xx_osm.h>
   49#include <dev/aic7xxx/aic79xx_inline.h>
   50#include <dev/aic7xxx/aicasm/aicasm_insformat.h>
   51#endif
   52
   53
   54/***************************** Lookup Tables **********************************/
   55static const char *const ahd_chip_names[] =
   56{
   57        "NONE",
   58        "aic7901",
   59        "aic7902",
   60        "aic7901A"
   61};
   62static const u_int num_chip_names = ARRAY_SIZE(ahd_chip_names);
   63
   64/*
   65 * Hardware error codes.
   66 */
   67struct ahd_hard_error_entry {
   68        uint8_t errno;
   69        const char *errmesg;
   70};
   71
   72static const struct ahd_hard_error_entry ahd_hard_errors[] = {
   73        { DSCTMOUT,        "Discard Timer has timed out" },
   74        { ILLOPCODE,        "Illegal Opcode in sequencer program" },
   75        { SQPARERR,        "Sequencer Parity Error" },
   76        { DPARERR,        "Data-path Parity Error" },
   77        { MPARERR,        "Scratch or SCB Memory Parity Error" },
   78        { CIOPARERR,        "CIOBUS Parity Error" },
   79};
   80static const u_int num_errors = ARRAY_SIZE(ahd_hard_errors);
   81
   82static const struct ahd_phase_table_entry ahd_phase_table[] =
   83{
   84        { P_DATAOUT,        MSG_NOOP,                "in Data-out phase"        },
   85        { P_DATAIN,        MSG_INITIATOR_DET_ERR,        "in Data-in phase"        },
   86        { P_DATAOUT_DT,        MSG_NOOP,                "in DT Data-out phase"        },
   87        { P_DATAIN_DT,        MSG_INITIATOR_DET_ERR,        "in DT Data-in phase"        },
   88        { P_COMMAND,        MSG_NOOP,                "in Command phase"        },
   89        { P_MESGOUT,        MSG_NOOP,                "in Message-out phase"        },
   90        { P_STATUS,        MSG_INITIATOR_DET_ERR,        "in Status phase"        },
   91        { P_MESGIN,        MSG_PARITY_ERROR,        "in Message-in phase"        },
   92        { P_BUSFREE,        MSG_NOOP,                "while idle"                },
   93        { 0,                MSG_NOOP,                "in unknown phase"        }
   94};
   95
   96/*
   97 * In most cases we only wish to itterate over real phases, so
   98 * exclude the last element from the count.
   99 */
  100static const u_int num_phases = ARRAY_SIZE(ahd_phase_table) - 1;
  101
  102/* Our Sequencer Program */
  103#include "aic79xx_seq.h"
  104
  105/**************************** Function Declarations ***************************/
  106static void                ahd_handle_transmission_error(struct ahd_softc *ahd);
  107static void                ahd_handle_lqiphase_error(struct ahd_softc *ahd,
  108                                                  u_int lqistat1);
  109static int                ahd_handle_pkt_busfree(struct ahd_softc *ahd,
  110                                               u_int busfreetime);
  111static int                ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
  112static void                ahd_handle_proto_violation(struct ahd_softc *ahd);
  113static void                ahd_force_renegotiation(struct ahd_softc *ahd,
  114                                                struct ahd_devinfo *devinfo);
  115
  116static struct ahd_tmode_tstate*
  117                        ahd_alloc_tstate(struct ahd_softc *ahd,
  118                                         u_int scsi_id, char channel);
  119#ifdef AHD_TARGET_MODE
  120static void                ahd_free_tstate(struct ahd_softc *ahd,
  121                                        u_int scsi_id, char channel, int force);
  122#endif
  123static void                ahd_devlimited_syncrate(struct ahd_softc *ahd,
  124                                                struct ahd_initiator_tinfo *,
  125                                                u_int *period,
  126                                                u_int *ppr_options,
  127                                                role_t role);
  128static void                ahd_update_neg_table(struct ahd_softc *ahd,
  129                                             struct ahd_devinfo *devinfo,
  130                                             struct ahd_transinfo *tinfo);
  131static void                ahd_update_pending_scbs(struct ahd_softc *ahd);
  132static void                ahd_fetch_devinfo(struct ahd_softc *ahd,
  133                                          struct ahd_devinfo *devinfo);
  134static void                ahd_scb_devinfo(struct ahd_softc *ahd,
  135                                        struct ahd_devinfo *devinfo,
  136                                        struct scb *scb);
  137static void                ahd_setup_initiator_msgout(struct ahd_softc *ahd,
  138                                                   struct ahd_devinfo *devinfo,
  139                                                   struct scb *scb);
  140static void                ahd_build_transfer_msg(struct ahd_softc *ahd,
  141                                               struct ahd_devinfo *devinfo);
  142static void                ahd_construct_sdtr(struct ahd_softc *ahd,
  143                                           struct ahd_devinfo *devinfo,
  144                                           u_int period, u_int offset);
  145static void                ahd_construct_wdtr(struct ahd_softc *ahd,
  146                                           struct ahd_devinfo *devinfo,
  147                                           u_int bus_width);
  148static void                ahd_construct_ppr(struct ahd_softc *ahd,
  149                                          struct ahd_devinfo *devinfo,
  150                                          u_int period, u_int offset,
  151                                          u_int bus_width, u_int ppr_options);
  152static void                ahd_clear_msg_state(struct ahd_softc *ahd);
  153static void                ahd_handle_message_phase(struct ahd_softc *ahd);
  154typedef enum {
  155        AHDMSG_1B,
  156        AHDMSG_2B,
  157        AHDMSG_EXT
  158} ahd_msgtype;
  159static int                ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
  160                                     u_int msgval, int full);
  161static int                ahd_parse_msg(struct ahd_softc *ahd,
  162                                      struct ahd_devinfo *devinfo);
  163static int                ahd_handle_msg_reject(struct ahd_softc *ahd,
  164                                              struct ahd_devinfo *devinfo);
  165static void                ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
  166                                                struct ahd_devinfo *devinfo);
  167static void                ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
  168static void                ahd_handle_devreset(struct ahd_softc *ahd,
  169                                            struct ahd_devinfo *devinfo,
  170                                            u_int lun, cam_status status,
  171                                            char *message, int verbose_level);
  172#ifdef AHD_TARGET_MODE
  173static void                ahd_setup_target_msgin(struct ahd_softc *ahd,
  174                                               struct ahd_devinfo *devinfo,
  175                                               struct scb *scb);
  176#endif
  177
  178static u_int                ahd_sglist_size(struct ahd_softc *ahd);
  179static u_int                ahd_sglist_allocsize(struct ahd_softc *ahd);
  180static bus_dmamap_callback_t
  181                        ahd_dmamap_cb; 
  182static void                ahd_initialize_hscbs(struct ahd_softc *ahd);
  183static int                ahd_init_scbdata(struct ahd_softc *ahd);
  184static void                ahd_fini_scbdata(struct ahd_softc *ahd);
  185static void                ahd_setup_iocell_workaround(struct ahd_softc *ahd);
  186static void                ahd_iocell_first_selection(struct ahd_softc *ahd);
  187static void                ahd_add_col_list(struct ahd_softc *ahd,
  188                                         struct scb *scb, u_int col_idx);
  189static void                ahd_rem_col_list(struct ahd_softc *ahd,
  190                                         struct scb *scb);
  191static void                ahd_chip_init(struct ahd_softc *ahd);
  192static void                ahd_qinfifo_requeue(struct ahd_softc *ahd,
  193                                            struct scb *prev_scb,
  194                                            struct scb *scb);
  195static int                ahd_qinfifo_count(struct ahd_softc *ahd);
  196static int                ahd_search_scb_list(struct ahd_softc *ahd, int target,
  197                                            char channel, int lun, u_int tag,
  198                                            role_t role, uint32_t status,
  199                                            ahd_search_action action,
  200                                            u_int *list_head, u_int *list_tail,
  201                                            u_int tid);
  202static void                ahd_stitch_tid_list(struct ahd_softc *ahd,
  203                                            u_int tid_prev, u_int tid_cur,
  204                                            u_int tid_next);
  205static void                ahd_add_scb_to_free_list(struct ahd_softc *ahd,
  206                                                 u_int scbid);
  207static u_int                ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
  208                                     u_int prev, u_int next, u_int tid);
  209static void                ahd_reset_current_bus(struct ahd_softc *ahd);
  210static ahd_callback_t        ahd_stat_timer;
  211#ifdef AHD_DUMP_SEQ
  212static void                ahd_dumpseq(struct ahd_softc *ahd);
  213#endif
  214static void                ahd_loadseq(struct ahd_softc *ahd);
  215static int                ahd_check_patch(struct ahd_softc *ahd,
  216                                        const struct patch **start_patch,
  217                                        u_int start_instr, u_int *skip_addr);
  218static u_int                ahd_resolve_seqaddr(struct ahd_softc *ahd,
  219                                            u_int address);
  220static void                ahd_download_instr(struct ahd_softc *ahd,
  221                                           u_int instrptr, uint8_t *dconsts);
  222static int                ahd_probe_stack_size(struct ahd_softc *ahd);
  223static int                ahd_scb_active_in_fifo(struct ahd_softc *ahd,
  224                                               struct scb *scb);
  225static void                ahd_run_data_fifo(struct ahd_softc *ahd,
  226                                          struct scb *scb);
  227
  228#ifdef AHD_TARGET_MODE
  229static void                ahd_queue_lstate_event(struct ahd_softc *ahd,
  230                                               struct ahd_tmode_lstate *lstate,
  231                                               u_int initiator_id,
  232                                               u_int event_type,
  233                                               u_int event_arg);
  234static void                ahd_update_scsiid(struct ahd_softc *ahd,
  235                                          u_int targid_mask);
  236static int                ahd_handle_target_cmd(struct ahd_softc *ahd,
  237                                              struct target_cmd *cmd);
  238#endif
  239
  240static int                ahd_abort_scbs(struct ahd_softc *ahd, int target,
  241                                       char channel, int lun, u_int tag,
  242                                       role_t role, uint32_t status);
  243static void                ahd_alloc_scbs(struct ahd_softc *ahd);
  244static void                ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl,
  245                                     u_int scbid);
  246static void                ahd_calc_residual(struct ahd_softc *ahd,
  247                                          struct scb *scb);
  248static void                ahd_clear_critical_section(struct ahd_softc *ahd);
  249static void                ahd_clear_intstat(struct ahd_softc *ahd);
  250static void                ahd_enable_coalescing(struct ahd_softc *ahd,
  251                                              int enable);
  252static u_int                ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl);
  253static void                ahd_freeze_devq(struct ahd_softc *ahd,
  254                                        struct scb *scb);
  255static void                ahd_handle_scb_status(struct ahd_softc *ahd,
  256                                              struct scb *scb);
  257static const struct ahd_phase_table_entry* ahd_lookup_phase_entry(int phase);
  258static void                ahd_shutdown(void *arg);
  259static void                ahd_update_coalescing_values(struct ahd_softc *ahd,
  260                                                     u_int timer,
  261                                                     u_int maxcmds,
  262                                                     u_int mincmds);
  263static int                ahd_verify_vpd_cksum(struct vpd_config *vpd);
  264static int                ahd_wait_seeprom(struct ahd_softc *ahd);
  265static int                ahd_match_scb(struct ahd_softc *ahd, struct scb *scb,
  266                                      int target, char channel, int lun,
  267                                      u_int tag, role_t role);
  268
  269static void                ahd_reset_cmds_pending(struct ahd_softc *ahd);
  270
  271/*************************** Interrupt Services *******************************/
  272static void                ahd_run_qoutfifo(struct ahd_softc *ahd);
  273#ifdef AHD_TARGET_MODE
  274static void                ahd_run_tqinfifo(struct ahd_softc *ahd, int paused);
  275#endif
  276static void                ahd_handle_hwerrint(struct ahd_softc *ahd);
  277static void                ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat);
  278static void                ahd_handle_scsiint(struct ahd_softc *ahd,
  279                                           u_int intstat);
  280
  281/************************ Sequencer Execution Control *************************/
  282void
  283ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst)
  284{
  285        if (ahd->src_mode == src && ahd->dst_mode == dst)
  286                return;
  287#ifdef AHD_DEBUG
  288        if (ahd->src_mode == AHD_MODE_UNKNOWN
  289         || ahd->dst_mode == AHD_MODE_UNKNOWN)
  290                panic("Setting mode prior to saving it.\n");
  291        if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
  292                printf("%s: Setting mode 0x%x\n", ahd_name(ahd),
  293                       ahd_build_mode_state(ahd, src, dst));
  294#endif
  295        ahd_outb(ahd, MODE_PTR, ahd_build_mode_state(ahd, src, dst));
  296        ahd->src_mode = src;
  297        ahd->dst_mode = dst;
  298}
  299
  300static void
  301ahd_update_modes(struct ahd_softc *ahd)
  302{
  303        ahd_mode_state mode_ptr;
  304        ahd_mode src;
  305        ahd_mode dst;
  306
  307        mode_ptr = ahd_inb(ahd, MODE_PTR);
  308#ifdef AHD_DEBUG
  309        if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
  310                printf("Reading mode 0x%x\n", mode_ptr);
  311#endif
  312        ahd_extract_mode_state(ahd, mode_ptr, &src, &dst);
  313        ahd_known_modes(ahd, src, dst);
  314}
  315
  316static void
  317ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode,
  318                 ahd_mode dstmode, const char *file, int line)
  319{
  320#ifdef AHD_DEBUG
  321        if ((srcmode & AHD_MK_MSK(ahd->src_mode)) == 0
  322         || (dstmode & AHD_MK_MSK(ahd->dst_mode)) == 0) {
  323                panic("%s:%s:%d: Mode assertion failed.\n",
  324                       ahd_name(ahd), file, line);
  325        }
  326#endif
  327}
  328
  329#define AHD_ASSERT_MODES(ahd, source, dest) \
  330        ahd_assert_modes(ahd, source, dest, __FILE__, __LINE__);
  331
  332ahd_mode_state
  333ahd_save_modes(struct ahd_softc *ahd)
  334{
  335        if (ahd->src_mode == AHD_MODE_UNKNOWN
  336         || ahd->dst_mode == AHD_MODE_UNKNOWN)
  337                ahd_update_modes(ahd);
  338
  339        return (ahd_build_mode_state(ahd, ahd->src_mode, ahd->dst_mode));
  340}
  341
  342void
  343ahd_restore_modes(struct ahd_softc *ahd, ahd_mode_state state)
  344{
  345        ahd_mode src;
  346        ahd_mode dst;
  347
  348        ahd_extract_mode_state(ahd, state, &src, &dst);
  349        ahd_set_modes(ahd, src, dst);
  350}
  351
  352/*
  353 * Determine whether the sequencer has halted code execution.
  354 * Returns non-zero status if the sequencer is stopped.
  355 */
  356int
  357ahd_is_paused(struct ahd_softc *ahd)
  358{
  359        return ((ahd_inb(ahd, HCNTRL) & PAUSE) != 0);
  360}
  361
  362/*
  363 * Request that the sequencer stop and wait, indefinitely, for it
  364 * to stop.  The sequencer will only acknowledge that it is paused
  365 * once it has reached an instruction boundary and PAUSEDIS is
  366 * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
  367 * for critical sections.
  368 */
  369void
  370ahd_pause(struct ahd_softc *ahd)
  371{
  372        ahd_outb(ahd, HCNTRL, ahd->pause);
  373
  374        /*
  375         * Since the sequencer can disable pausing in a critical section, we
  376         * must loop until it actually stops.
  377         */
  378        while (ahd_is_paused(ahd) == 0)
  379                ;
  380}
  381
  382/*
  383 * Allow the sequencer to continue program execution.
  384 * We check here to ensure that no additional interrupt
  385 * sources that would cause the sequencer to halt have been
  386 * asserted.  If, for example, a SCSI bus reset is detected
  387 * while we are fielding a different, pausing, interrupt type,
  388 * we don't want to release the sequencer before going back
  389 * into our interrupt handler and dealing with this new
  390 * condition.
  391 */
  392void
  393ahd_unpause(struct ahd_softc *ahd)
  394{
  395        /*
  396         * Automatically restore our modes to those saved
  397         * prior to the first change of the mode.
  398         */
  399        if (ahd->saved_src_mode != AHD_MODE_UNKNOWN
  400         && ahd->saved_dst_mode != AHD_MODE_UNKNOWN) {
  401                if ((ahd->flags & AHD_UPDATE_PEND_CMDS) != 0)
  402                        ahd_reset_cmds_pending(ahd);
  403                ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
  404        }
  405
  406        if ((ahd_inb(ahd, INTSTAT) & ~CMDCMPLT) == 0)
  407                ahd_outb(ahd, HCNTRL, ahd->unpause);
  408
  409        ahd_known_modes(ahd, AHD_MODE_UNKNOWN, AHD_MODE_UNKNOWN);
  410}
  411
  412/*********************** Scatter Gather List Handling *************************/
  413void *
  414ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb,
  415             void *sgptr, dma_addr_t addr, bus_size_t len, int last)
  416{
  417        scb->sg_count++;
  418        if (sizeof(dma_addr_t) > 4
  419         && (ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
  420                struct ahd_dma64_seg *sg;
  421
  422                sg = (struct ahd_dma64_seg *)sgptr;
  423                sg->addr = ahd_htole64(addr);
  424                sg->len = ahd_htole32(len | (last ? AHD_DMA_LAST_SEG : 0));
  425                return (sg + 1);
  426        } else {
  427                struct ahd_dma_seg *sg;
  428
  429                sg = (struct ahd_dma_seg *)sgptr;
  430                sg->addr = ahd_htole32(addr & 0xFFFFFFFF);
  431                sg->len = ahd_htole32(len | ((addr >> 8) & 0x7F000000)
  432                                    | (last ? AHD_DMA_LAST_SEG : 0));
  433                return (sg + 1);
  434        }
  435}
  436
  437static void
  438ahd_setup_scb_common(struct ahd_softc *ahd, struct scb *scb)
  439{
  440        /* XXX Handle target mode SCBs. */
  441        scb->crc_retry_count = 0;
  442        if ((scb->flags & SCB_PACKETIZED) != 0) {
  443                /* XXX what about ACA??  It is type 4, but TAG_TYPE == 0x3. */
  444                scb->hscb->task_attribute = scb->hscb->control & SCB_TAG_TYPE;
  445        } else {
  446                if (ahd_get_transfer_length(scb) & 0x01)
  447                        scb->hscb->task_attribute = SCB_XFERLEN_ODD;
  448                else
  449                        scb->hscb->task_attribute = 0;
  450        }
  451
  452        if (scb->hscb->cdb_len <= MAX_CDB_LEN_WITH_SENSE_ADDR
  453         || (scb->hscb->cdb_len & SCB_CDB_LEN_PTR) != 0)
  454                scb->hscb->shared_data.idata.cdb_plus_saddr.sense_addr =
  455                    ahd_htole32(scb->sense_busaddr);
  456}
  457
  458static void
  459ahd_setup_data_scb(struct ahd_softc *ahd, struct scb *scb)
  460{
  461        /*
  462         * Copy the first SG into the "current" data ponter area.
  463         */
  464        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
  465                struct ahd_dma64_seg *sg;
  466
  467                sg = (struct ahd_dma64_seg *)scb->sg_list;
  468                scb->hscb->dataptr = sg->addr;
  469                scb->hscb->datacnt = sg->len;
  470        } else {
  471                struct ahd_dma_seg *sg;
  472                uint32_t *dataptr_words;
  473
  474                sg = (struct ahd_dma_seg *)scb->sg_list;
  475                dataptr_words = (uint32_t*)&scb->hscb->dataptr;
  476                dataptr_words[0] = sg->addr;
  477                dataptr_words[1] = 0;
  478                if ((ahd->flags & AHD_39BIT_ADDRESSING) != 0) {
  479                        uint64_t high_addr;
  480
  481                        high_addr = ahd_le32toh(sg->len) & 0x7F000000;
  482                        scb->hscb->dataptr |= ahd_htole64(high_addr << 8);
  483                }
  484                scb->hscb->datacnt = sg->len;
  485        }
  486        /*
  487         * Note where to find the SG entries in bus space.
  488         * We also set the full residual flag which the
  489         * sequencer will clear as soon as a data transfer
  490         * occurs.
  491         */
  492        scb->hscb->sgptr = ahd_htole32(scb->sg_list_busaddr|SG_FULL_RESID);
  493}
  494
  495static void
  496ahd_setup_noxfer_scb(struct ahd_softc *ahd, struct scb *scb)
  497{
  498        scb->hscb->sgptr = ahd_htole32(SG_LIST_NULL);
  499        scb->hscb->dataptr = 0;
  500        scb->hscb->datacnt = 0;
  501}
  502
  503/************************** Memory mapping routines ***************************/
  504static void *
  505ahd_sg_bus_to_virt(struct ahd_softc *ahd, struct scb *scb, uint32_t sg_busaddr)
  506{
  507        dma_addr_t sg_offset;
  508
  509        /* sg_list_phys points to entry 1, not 0 */
  510        sg_offset = sg_busaddr - (scb->sg_list_busaddr - ahd_sg_size(ahd));
  511        return ((uint8_t *)scb->sg_list + sg_offset);
  512}
  513
  514static uint32_t
  515ahd_sg_virt_to_bus(struct ahd_softc *ahd, struct scb *scb, void *sg)
  516{
  517        dma_addr_t sg_offset;
  518
  519        /* sg_list_phys points to entry 1, not 0 */
  520        sg_offset = ((uint8_t *)sg - (uint8_t *)scb->sg_list)
  521                  - ahd_sg_size(ahd);
  522
  523        return (scb->sg_list_busaddr + sg_offset);
  524}
  525
  526static void
  527ahd_sync_scb(struct ahd_softc *ahd, struct scb *scb, int op)
  528{
  529        ahd_dmamap_sync(ahd, ahd->scb_data.hscb_dmat,
  530                        scb->hscb_map->dmamap,
  531                        /*offset*/(uint8_t*)scb->hscb - scb->hscb_map->vaddr,
  532                        /*len*/sizeof(*scb->hscb), op);
  533}
  534
  535void
  536ahd_sync_sglist(struct ahd_softc *ahd, struct scb *scb, int op)
  537{
  538        if (scb->sg_count == 0)
  539                return;
  540
  541        ahd_dmamap_sync(ahd, ahd->scb_data.sg_dmat,
  542                        scb->sg_map->dmamap,
  543                        /*offset*/scb->sg_list_busaddr - ahd_sg_size(ahd),
  544                        /*len*/ahd_sg_size(ahd) * scb->sg_count, op);
  545}
  546
  547static void
  548ahd_sync_sense(struct ahd_softc *ahd, struct scb *scb, int op)
  549{
  550        ahd_dmamap_sync(ahd, ahd->scb_data.sense_dmat,
  551                        scb->sense_map->dmamap,
  552                        /*offset*/scb->sense_busaddr,
  553                        /*len*/AHD_SENSE_BUFSIZE, op);
  554}
  555
  556#ifdef AHD_TARGET_MODE
  557static uint32_t
  558ahd_targetcmd_offset(struct ahd_softc *ahd, u_int index)
  559{
  560        return (((uint8_t *)&ahd->targetcmds[index])
  561               - (uint8_t *)ahd->qoutfifo);
  562}
  563#endif
  564
  565/*********************** Miscelaneous Support Functions ***********************/
  566/*
  567 * Return pointers to the transfer negotiation information
  568 * for the specified our_id/remote_id pair.
  569 */
  570struct ahd_initiator_tinfo *
  571ahd_fetch_transinfo(struct ahd_softc *ahd, char channel, u_int our_id,
  572                    u_int remote_id, struct ahd_tmode_tstate **tstate)
  573{
  574        /*
  575         * Transfer data structures are stored from the perspective
  576         * of the target role.  Since the parameters for a connection
  577         * in the initiator role to a given target are the same as
  578         * when the roles are reversed, we pretend we are the target.
  579         */
  580        if (channel == 'B')
  581                our_id += 8;
  582        *tstate = ahd->enabled_targets[our_id];
  583        return (&(*tstate)->transinfo[remote_id]);
  584}
  585
  586uint16_t
  587ahd_inw(struct ahd_softc *ahd, u_int port)
  588{
  589        /*
  590         * Read high byte first as some registers increment
  591         * or have other side effects when the low byte is
  592         * read.
  593         */
  594        uint16_t r = ahd_inb(ahd, port+1) << 8;
  595        return r | ahd_inb(ahd, port);
  596}
  597
  598void
  599ahd_outw(struct ahd_softc *ahd, u_int port, u_int value)
  600{
  601        /*
  602         * Write low byte first to accomodate registers
  603         * such as PRGMCNT where the order maters.
  604         */
  605        ahd_outb(ahd, port, value & 0xFF);
  606        ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
  607}
  608
  609uint32_t
  610ahd_inl(struct ahd_softc *ahd, u_int port)
  611{
  612        return ((ahd_inb(ahd, port))
  613              | (ahd_inb(ahd, port+1) << 8)
  614              | (ahd_inb(ahd, port+2) << 16)
  615              | (ahd_inb(ahd, port+3) << 24));
  616}
  617
  618void
  619ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value)
  620{
  621        ahd_outb(ahd, port, (value) & 0xFF);
  622        ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF);
  623        ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF);
  624        ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF);
  625}
  626
  627uint64_t
  628ahd_inq(struct ahd_softc *ahd, u_int port)
  629{
  630        return ((ahd_inb(ahd, port))
  631              | (ahd_inb(ahd, port+1) << 8)
  632              | (ahd_inb(ahd, port+2) << 16)
  633              | (ahd_inb(ahd, port+3) << 24)
  634              | (((uint64_t)ahd_inb(ahd, port+4)) << 32)
  635              | (((uint64_t)ahd_inb(ahd, port+5)) << 40)
  636              | (((uint64_t)ahd_inb(ahd, port+6)) << 48)
  637              | (((uint64_t)ahd_inb(ahd, port+7)) << 56));
  638}
  639
  640void
  641ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value)
  642{
  643        ahd_outb(ahd, port, value & 0xFF);
  644        ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
  645        ahd_outb(ahd, port+2, (value >> 16) & 0xFF);
  646        ahd_outb(ahd, port+3, (value >> 24) & 0xFF);
  647        ahd_outb(ahd, port+4, (value >> 32) & 0xFF);
  648        ahd_outb(ahd, port+5, (value >> 40) & 0xFF);
  649        ahd_outb(ahd, port+6, (value >> 48) & 0xFF);
  650        ahd_outb(ahd, port+7, (value >> 56) & 0xFF);
  651}
  652
  653u_int
  654ahd_get_scbptr(struct ahd_softc *ahd)
  655{
  656        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
  657                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
  658        return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8));
  659}
  660
  661void
  662ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr)
  663{
  664        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
  665                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
  666        ahd_outb(ahd, SCBPTR, scbptr & 0xFF);
  667        ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF);
  668}
  669
  670#if 0 /* unused */
  671static u_int
  672ahd_get_hnscb_qoff(struct ahd_softc *ahd)
  673{
  674        return (ahd_inw_atomic(ahd, HNSCB_QOFF));
  675}
  676#endif
  677
  678static void
  679ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value)
  680{
  681        ahd_outw_atomic(ahd, HNSCB_QOFF, value);
  682}
  683
  684#if 0 /* unused */
  685static u_int
  686ahd_get_hescb_qoff(struct ahd_softc *ahd)
  687{
  688        return (ahd_inb(ahd, HESCB_QOFF));
  689}
  690#endif
  691
  692static void
  693ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value)
  694{
  695        ahd_outb(ahd, HESCB_QOFF, value);
  696}
  697
  698static u_int
  699ahd_get_snscb_qoff(struct ahd_softc *ahd)
  700{
  701        u_int oldvalue;
  702
  703        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
  704        oldvalue = ahd_inw(ahd, SNSCB_QOFF);
  705        ahd_outw(ahd, SNSCB_QOFF, oldvalue);
  706        return (oldvalue);
  707}
  708
  709static void
  710ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value)
  711{
  712        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
  713        ahd_outw(ahd, SNSCB_QOFF, value);
  714}
  715
  716#if 0 /* unused */
  717static u_int
  718ahd_get_sescb_qoff(struct ahd_softc *ahd)
  719{
  720        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
  721        return (ahd_inb(ahd, SESCB_QOFF));
  722}
  723#endif
  724
  725static void
  726ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value)
  727{
  728        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
  729        ahd_outb(ahd, SESCB_QOFF, value);
  730}
  731
  732#if 0 /* unused */
  733static u_int
  734ahd_get_sdscb_qoff(struct ahd_softc *ahd)
  735{
  736        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
  737        return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8));
  738}
  739#endif
  740
  741static void
  742ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value)
  743{
  744        AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
  745        ahd_outb(ahd, SDSCB_QOFF, value & 0xFF);
  746        ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF);
  747}
  748
  749u_int
  750ahd_inb_scbram(struct ahd_softc *ahd, u_int offset)
  751{
  752        u_int value;
  753
  754        /*
  755         * Workaround PCI-X Rev A. hardware bug.
  756         * After a host read of SCB memory, the chip
  757         * may become confused into thinking prefetch
  758         * was required.  This starts the discard timer
  759         * running and can cause an unexpected discard
  760         * timer interrupt.  The work around is to read
  761         * a normal register prior to the exhaustion of
  762         * the discard timer.  The mode pointer register
  763         * has no side effects and so serves well for
  764         * this purpose.
  765         *
  766         * Razor #528
  767         */
  768        value = ahd_inb(ahd, offset);
  769        if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0)
  770                ahd_inb(ahd, MODE_PTR);
  771        return (value);
  772}
  773
  774u_int
  775ahd_inw_scbram(struct ahd_softc *ahd, u_int offset)
  776{
  777        return (ahd_inb_scbram(ahd, offset)
  778              | (ahd_inb_scbram(ahd, offset+1) << 8));
  779}
  780
  781static uint32_t
  782ahd_inl_scbram(struct ahd_softc *ahd, u_int offset)
  783{
  784        return (ahd_inw_scbram(ahd, offset)
  785              | (ahd_inw_scbram(ahd, offset+2) << 16));
  786}
  787
  788static uint64_t
  789ahd_inq_scbram(struct ahd_softc *ahd, u_int offset)
  790{
  791        return (ahd_inl_scbram(ahd, offset)
  792              | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32);
  793}
  794
  795struct scb *
  796ahd_lookup_scb(struct ahd_softc *ahd, u_int tag)
  797{
  798        struct scb* scb;
  799
  800        if (tag >= AHD_SCB_MAX)
  801                return (NULL);
  802        scb = ahd->scb_data.scbindex[tag];
  803        if (scb != NULL)
  804                ahd_sync_scb(ahd, scb,
  805                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
  806        return (scb);
  807}
  808
  809static void
  810ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb)
  811{
  812        struct         hardware_scb *q_hscb;
  813        struct         map_node *q_hscb_map;
  814        uint32_t saved_hscb_busaddr;
  815
  816        /*
  817         * Our queuing method is a bit tricky.  The card
  818         * knows in advance which HSCB (by address) to download,
  819         * and we can't disappoint it.  To achieve this, the next
  820         * HSCB to download is saved off in ahd->next_queued_hscb.
  821         * When we are called to queue "an arbitrary scb",
  822         * we copy the contents of the incoming HSCB to the one
  823         * the sequencer knows about, swap HSCB pointers and
  824         * finally assign the SCB to the tag indexed location
  825         * in the scb_array.  This makes sure that we can still
  826         * locate the correct SCB by SCB_TAG.
  827         */
  828        q_hscb = ahd->next_queued_hscb;
  829        q_hscb_map = ahd->next_queued_hscb_map;
  830        saved_hscb_busaddr = q_hscb->hscb_busaddr;
  831        memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
  832        q_hscb->hscb_busaddr = saved_hscb_busaddr;
  833        q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
  834
  835        /* Now swap HSCB pointers. */
  836        ahd->next_queued_hscb = scb->hscb;
  837        ahd->next_queued_hscb_map = scb->hscb_map;
  838        scb->hscb = q_hscb;
  839        scb->hscb_map = q_hscb_map;
  840
  841        /* Now define the mapping from tag to SCB in the scbindex */
  842        ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb;
  843}
  844
  845/*
  846 * Tell the sequencer about a new transaction to execute.
  847 */
  848void
  849ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb)
  850{
  851        ahd_swap_with_next_hscb(ahd, scb);
  852
  853        if (SCBID_IS_NULL(SCB_GET_TAG(scb)))
  854                panic("Attempt to queue invalid SCB tag %x\n",
  855                      SCB_GET_TAG(scb));
  856
  857        /*
  858         * Keep a history of SCBs we've downloaded in the qinfifo.
  859         */
  860        ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
  861        ahd->qinfifonext++;
  862
  863        if (scb->sg_count != 0)
  864                ahd_setup_data_scb(ahd, scb);
  865        else
  866                ahd_setup_noxfer_scb(ahd, scb);
  867        ahd_setup_scb_common(ahd, scb);
  868
  869        /*
  870         * Make sure our data is consistent from the
  871         * perspective of the adapter.
  872         */
  873        ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
  874
  875#ifdef AHD_DEBUG
  876        if ((ahd_debug & AHD_SHOW_QUEUE) != 0) {
  877                uint64_t host_dataptr;
  878
  879                host_dataptr = ahd_le64toh(scb->hscb->dataptr);
  880                printf("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n",
  881                       ahd_name(ahd),
  882                       SCB_GET_TAG(scb), scb->hscb->scsiid,
  883                       ahd_le32toh(scb->hscb->hscb_busaddr),
  884                       (u_int)((host_dataptr >> 32) & 0xFFFFFFFF),
  885                       (u_int)(host_dataptr & 0xFFFFFFFF),
  886                       ahd_le32toh(scb->hscb->datacnt));
  887        }
  888#endif
  889        /* Tell the adapter about the newly queued SCB */
  890        ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
  891}
  892
  893/************************** Interrupt Processing ******************************/
  894static void
  895ahd_sync_qoutfifo(struct ahd_softc *ahd, int op)
  896{
  897        ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
  898                        /*offset*/0,
  899                        /*len*/AHD_SCB_MAX * sizeof(struct ahd_completion), op);
  900}
  901
  902static void
  903ahd_sync_tqinfifo(struct ahd_softc *ahd, int op)
  904{
  905#ifdef AHD_TARGET_MODE
  906        if ((ahd->flags & AHD_TARGETROLE) != 0) {
  907                ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
  908                                ahd->shared_data_map.dmamap,
  909                                ahd_targetcmd_offset(ahd, 0),
  910                                sizeof(struct target_cmd) * AHD_TMODE_CMDS,
  911                                op);
  912        }
  913#endif
  914}
  915
  916/*
  917 * See if the firmware has posted any completed commands
  918 * into our in-core command complete fifos.
  919 */
  920#define AHD_RUN_QOUTFIFO 0x1
  921#define AHD_RUN_TQINFIFO 0x2
  922static u_int
  923ahd_check_cmdcmpltqueues(struct ahd_softc *ahd)
  924{
  925        u_int retval;
  926
  927        retval = 0;
  928        ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
  929                        /*offset*/ahd->qoutfifonext * sizeof(*ahd->qoutfifo),
  930                        /*len*/sizeof(*ahd->qoutfifo), BUS_DMASYNC_POSTREAD);
  931        if (ahd->qoutfifo[ahd->qoutfifonext].valid_tag
  932          == ahd->qoutfifonext_valid_tag)
  933                retval |= AHD_RUN_QOUTFIFO;
  934#ifdef AHD_TARGET_MODE
  935        if ((ahd->flags & AHD_TARGETROLE) != 0
  936         && (ahd->flags & AHD_TQINFIFO_BLOCKED) == 0) {
  937                ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
  938                                ahd->shared_data_map.dmamap,
  939                                ahd_targetcmd_offset(ahd, ahd->tqinfifofnext),
  940                                /*len*/sizeof(struct target_cmd),
  941                                BUS_DMASYNC_POSTREAD);
  942                if (ahd->targetcmds[ahd->tqinfifonext].cmd_valid != 0)
  943                        retval |= AHD_RUN_TQINFIFO;
  944        }
  945#endif
  946        return (retval);
  947}
  948
  949/*
  950 * Catch an interrupt from the adapter
  951 */
  952int
  953ahd_intr(struct ahd_softc *ahd)
  954{
  955        u_int        intstat;
  956
  957        if ((ahd->pause & INTEN) == 0) {
  958                /*
  959                 * Our interrupt is not enabled on the chip
  960                 * and may be disabled for re-entrancy reasons,
  961                 * so just return.  This is likely just a shared
  962                 * interrupt.
  963                 */
  964                return (0);
  965        }
  966
  967        /*
  968         * Instead of directly reading the interrupt status register,
  969         * infer the cause of the interrupt by checking our in-core
  970         * completion queues.  This avoids a costly PCI bus read in
  971         * most cases.
  972         */
  973        if ((ahd->flags & AHD_ALL_INTERRUPTS) == 0
  974         && (ahd_check_cmdcmpltqueues(ahd) != 0))
  975                intstat = CMDCMPLT;
  976        else
  977                intstat = ahd_inb(ahd, INTSTAT);
  978
  979        if ((intstat & INT_PEND) == 0)
  980                return (0);
  981
  982        if (intstat & CMDCMPLT) {
  983                ahd_outb(ahd, CLRINT, CLRCMDINT);
  984
  985                /*
  986                 * Ensure that the chip sees that we've cleared
  987                 * this interrupt before we walk the output fifo.
  988                 * Otherwise, we may, due to posted bus writes,
  989                 * clear the interrupt after we finish the scan,
  990                 * and after the sequencer has added new entries
  991                 * and asserted the interrupt again.
  992                 */
  993                if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
  994                        if (ahd_is_paused(ahd)) {
  995                                /*
  996                                 * Potentially lost SEQINT.
  997                                 * If SEQINTCODE is non-zero,
  998                                 * simulate the SEQINT.
  999                                 */
 1000                                if (ahd_inb(ahd, SEQINTCODE) != NO_SEQINT)
 1001                                        intstat |= SEQINT;
 1002                        }
 1003                } else {
 1004                        ahd_flush_device_writes(ahd);
 1005                }
 1006                ahd_run_qoutfifo(ahd);
 1007                ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket]++;
 1008                ahd->cmdcmplt_total++;
 1009#ifdef AHD_TARGET_MODE
 1010                if ((ahd->flags & AHD_TARGETROLE) != 0)
 1011                        ahd_run_tqinfifo(ahd, /*paused*/FALSE);
 1012#endif
 1013        }
 1014
 1015        /*
 1016         * Handle statuses that may invalidate our cached
 1017         * copy of INTSTAT separately.
 1018         */
 1019        if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0) {
 1020                /* Hot eject.  Do nothing */
 1021        } else if (intstat & HWERRINT) {
 1022                ahd_handle_hwerrint(ahd);
 1023        } else if ((intstat & (PCIINT|SPLTINT)) != 0) {
 1024                ahd->bus_intr(ahd);
 1025        } else {
 1026
 1027                if ((intstat & SEQINT) != 0)
 1028                        ahd_handle_seqint(ahd, intstat);
 1029
 1030                if ((intstat & SCSIINT) != 0)
 1031                        ahd_handle_scsiint(ahd, intstat);
 1032        }
 1033        return (1);
 1034}
 1035
 1036/******************************** Private Inlines *****************************/
 1037static __inline void
 1038ahd_assert_atn(struct ahd_softc *ahd)
 1039{
 1040        ahd_outb(ahd, SCSISIGO, ATNO);
 1041}
 1042
 1043/*
 1044 * Determine if the current connection has a packetized
 1045 * agreement.  This does not necessarily mean that we
 1046 * are currently in a packetized transfer.  We could
 1047 * just as easily be sending or receiving a message.
 1048 */
 1049static int
 1050ahd_currently_packetized(struct ahd_softc *ahd)
 1051{
 1052        ahd_mode_state         saved_modes;
 1053        int                 packetized;
 1054
 1055        saved_modes = ahd_save_modes(ahd);
 1056        if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
 1057                /*
 1058                 * The packetized bit refers to the last
 1059                 * connection, not the current one.  Check
 1060                 * for non-zero LQISTATE instead.
 1061                 */
 1062                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 1063                packetized = ahd_inb(ahd, LQISTATE) != 0;
 1064        } else {
 1065                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 1066                packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
 1067        }
 1068        ahd_restore_modes(ahd, saved_modes);
 1069        return (packetized);
 1070}
 1071
 1072static __inline int
 1073ahd_set_active_fifo(struct ahd_softc *ahd)
 1074{
 1075        u_int active_fifo;
 1076
 1077        AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 1078        active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
 1079        switch (active_fifo) {
 1080        case 0:
 1081        case 1:
 1082                ahd_set_modes(ahd, active_fifo, active_fifo);
 1083                return (1);
 1084        default:
 1085                return (0);
 1086        }
 1087}
 1088
 1089static __inline void
 1090ahd_unbusy_tcl(struct ahd_softc *ahd, u_int tcl)
 1091{
 1092        ahd_busy_tcl(ahd, tcl, SCB_LIST_NULL);
 1093}
 1094
 1095/*
 1096 * Determine whether the sequencer reported a residual
 1097 * for this SCB/transaction.
 1098 */
 1099static __inline void
 1100ahd_update_residual(struct ahd_softc *ahd, struct scb *scb)
 1101{
 1102        uint32_t sgptr;
 1103
 1104        sgptr = ahd_le32toh(scb->hscb->sgptr);
 1105        if ((sgptr & SG_STATUS_VALID) != 0)
 1106                ahd_calc_residual(ahd, scb);
 1107}
 1108
 1109static __inline void
 1110ahd_complete_scb(struct ahd_softc *ahd, struct scb *scb)
 1111{
 1112        uint32_t sgptr;
 1113
 1114        sgptr = ahd_le32toh(scb->hscb->sgptr);
 1115        if ((sgptr & SG_STATUS_VALID) != 0)
 1116                ahd_handle_scb_status(ahd, scb);
 1117        else
 1118                ahd_done(ahd, scb);
 1119}
 1120
 1121
 1122/************************* Sequencer Execution Control ************************/
 1123/*
 1124 * Restart the sequencer program from address zero
 1125 */
 1126static void
 1127ahd_restart(struct ahd_softc *ahd)
 1128{
 1129
 1130        ahd_pause(ahd);
 1131
 1132        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 1133
 1134        /* No more pending messages */
 1135        ahd_clear_msg_state(ahd);
 1136        ahd_outb(ahd, SCSISIGO, 0);                /* De-assert BSY */
 1137        ahd_outb(ahd, MSG_OUT, MSG_NOOP);        /* No message to send */
 1138        ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
 1139        ahd_outb(ahd, SEQINTCTL, 0);
 1140        ahd_outb(ahd, LASTPHASE, P_BUSFREE);
 1141        ahd_outb(ahd, SEQ_FLAGS, 0);
 1142        ahd_outb(ahd, SAVED_SCSIID, 0xFF);
 1143        ahd_outb(ahd, SAVED_LUN, 0xFF);
 1144
 1145        /*
 1146         * Ensure that the sequencer's idea of TQINPOS
 1147         * matches our own.  The sequencer increments TQINPOS
 1148         * only after it sees a DMA complete and a reset could
 1149         * occur before the increment leaving the kernel to believe
 1150         * the command arrived but the sequencer to not.
 1151         */
 1152        ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
 1153
 1154        /* Always allow reselection */
 1155        ahd_outb(ahd, SCSISEQ1,
 1156                 ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 1157        ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 1158
 1159        /*
 1160         * Clear any pending sequencer interrupt.  It is no
 1161         * longer relevant since we're resetting the Program
 1162         * Counter.
 1163         */
 1164        ahd_outb(ahd, CLRINT, CLRSEQINT);
 1165
 1166        ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
 1167        ahd_unpause(ahd);
 1168}
 1169
 1170static void
 1171ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
 1172{
 1173        ahd_mode_state         saved_modes;
 1174
 1175#ifdef AHD_DEBUG
 1176        if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
 1177                printf("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
 1178#endif
 1179        saved_modes = ahd_save_modes(ahd);
 1180        ahd_set_modes(ahd, fifo, fifo);
 1181        ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
 1182        if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 1183                ahd_outb(ahd, CCSGCTL, CCSGRESET);
 1184        ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 1185        ahd_outb(ahd, SG_STATE, 0);
 1186        ahd_restore_modes(ahd, saved_modes);
 1187}
 1188
 1189/************************* Input/Output Queues ********************************/
 1190/*
 1191 * Flush and completed commands that are sitting in the command
 1192 * complete queues down on the chip but have yet to be dma'ed back up.
 1193 */
 1194static void
 1195ahd_flush_qoutfifo(struct ahd_softc *ahd)
 1196{
 1197        struct                scb *scb;
 1198        ahd_mode_state        saved_modes;
 1199        u_int                saved_scbptr;
 1200        u_int                ccscbctl;
 1201        u_int                scbid;
 1202        u_int                next_scbid;
 1203
 1204        saved_modes = ahd_save_modes(ahd);
 1205
 1206        /*
 1207         * Flush the good status FIFO for completed packetized commands.
 1208         */
 1209        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 1210        saved_scbptr = ahd_get_scbptr(ahd);
 1211        while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
 1212                u_int fifo_mode;
 1213                u_int i;
 1214                
 1215                scbid = ahd_inw(ahd, GSFIFO);
 1216                scb = ahd_lookup_scb(ahd, scbid);
 1217                if (scb == NULL) {
 1218                        printf("%s: Warning - GSFIFO SCB %d invalid\n",
 1219                               ahd_name(ahd), scbid);
 1220                        continue;
 1221                }
 1222                /*
 1223                 * Determine if this transaction is still active in
 1224                 * any FIFO.  If it is, we must flush that FIFO to
 1225                 * the host before completing the  command.
 1226                 */
 1227                fifo_mode = 0;
 1228rescan_fifos:
 1229                for (i = 0; i < 2; i++) {
 1230                        /* Toggle to the other mode. */
 1231                        fifo_mode ^= 1;
 1232                        ahd_set_modes(ahd, fifo_mode, fifo_mode);
 1233
 1234                        if (ahd_scb_active_in_fifo(ahd, scb) == 0)
 1235                                continue;
 1236
 1237                        ahd_run_data_fifo(ahd, scb);
 1238
 1239                        /*
 1240                         * Running this FIFO may cause a CFG4DATA for
 1241                         * this same transaction to assert in the other
 1242                         * FIFO or a new snapshot SAVEPTRS interrupt
 1243                         * in this FIFO.  Even running a FIFO may not
 1244                         * clear the transaction if we are still waiting
 1245                         * for data to drain to the host. We must loop
 1246                         * until the transaction is not active in either
 1247                         * FIFO just to be sure.  Reset our loop counter
 1248                         * so we will visit both FIFOs again before
 1249                         * declaring this transaction finished.  We
 1250                         * also delay a bit so that status has a chance
 1251                         * to change before we look at this FIFO again.
 1252                         */
 1253                        ahd_delay(200);
 1254                        goto rescan_fifos;
 1255                }
 1256                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 1257                ahd_set_scbptr(ahd, scbid);
 1258                if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
 1259                 && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
 1260                  || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
 1261                      & SG_LIST_NULL) != 0)) {
 1262                        u_int comp_head;
 1263
 1264                        /*
 1265                         * The transfer completed with a residual.
 1266                         * Place this SCB on the complete DMA list
 1267                         * so that we update our in-core copy of the
 1268                         * SCB before completing the command.
 1269                         */
 1270                        ahd_outb(ahd, SCB_SCSI_STATUS, 0);
 1271                        ahd_outb(ahd, SCB_SGPTR,
 1272                                 ahd_inb_scbram(ahd, SCB_SGPTR)
 1273                                 | SG_STATUS_VALID);
 1274                        ahd_outw(ahd, SCB_TAG, scbid);
 1275                        ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
 1276                        comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 1277                        if (SCBID_IS_NULL(comp_head)) {
 1278                                ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
 1279                                ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 1280                        } else {
 1281                                u_int tail;
 1282
 1283                                tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
 1284                                ahd_set_scbptr(ahd, tail);
 1285                                ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
 1286                                ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
 1287                                ahd_set_scbptr(ahd, scbid);
 1288                        }
 1289                } else
 1290                        ahd_complete_scb(ahd, scb);
 1291        }
 1292        ahd_set_scbptr(ahd, saved_scbptr);
 1293
 1294        /*
 1295         * Setup for command channel portion of flush.
 1296         */
 1297        ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
 1298
 1299        /*
 1300         * Wait for any inprogress DMA to complete and clear DMA state
 1301         * if this if for an SCB in the qinfifo.
 1302         */
 1303        while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
 1304
 1305                if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
 1306                        if ((ccscbctl & ARRDONE) != 0)
 1307                                break;
 1308                } else if ((ccscbctl & CCSCBDONE) != 0)
 1309                        break;
 1310                ahd_delay(200);
 1311        }
 1312        /*
 1313         * We leave the sequencer to cleanup in the case of DMA's to
 1314         * update the qoutfifo.  In all other cases (DMA's to the
 1315         * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
 1316         * we disable the DMA engine so that the sequencer will not
 1317         * attempt to handle the DMA completion.
 1318         */
 1319        if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
 1320                ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
 1321
 1322        /*
 1323         * Complete any SCBs that just finished
 1324         * being DMA'ed into the qoutfifo.
 1325         */
 1326        ahd_run_qoutfifo(ahd);
 1327
 1328        saved_scbptr = ahd_get_scbptr(ahd);
 1329        /*
 1330         * Manually update/complete any completed SCBs that are waiting to be
 1331         * DMA'ed back up to the host.
 1332         */
 1333        scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
 1334        while (!SCBID_IS_NULL(scbid)) {
 1335                uint8_t *hscb_ptr;
 1336                u_int         i;
 1337                
 1338                ahd_set_scbptr(ahd, scbid);
 1339                next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 1340                scb = ahd_lookup_scb(ahd, scbid);
 1341                if (scb == NULL) {
 1342                        printf("%s: Warning - DMA-up and complete "
 1343                               "SCB %d invalid\n", ahd_name(ahd), scbid);
 1344                        continue;
 1345                }
 1346                hscb_ptr = (uint8_t *)scb->hscb;
 1347                for (i = 0; i < sizeof(struct hardware_scb); i++)
 1348                        *hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
 1349
 1350                ahd_complete_scb(ahd, scb);
 1351                scbid = next_scbid;
 1352        }
 1353        ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
 1354        ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
 1355
 1356        scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
 1357        while (!SCBID_IS_NULL(scbid)) {
 1358
 1359                ahd_set_scbptr(ahd, scbid);
 1360                next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 1361                scb = ahd_lookup_scb(ahd, scbid);
 1362                if (scb == NULL) {
 1363                        printf("%s: Warning - Complete Qfrz SCB %d invalid\n",
 1364                               ahd_name(ahd), scbid);
 1365                        continue;
 1366                }
 1367
 1368                ahd_complete_scb(ahd, scb);
 1369                scbid = next_scbid;
 1370        }
 1371        ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
 1372
 1373        scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
 1374        while (!SCBID_IS_NULL(scbid)) {
 1375
 1376                ahd_set_scbptr(ahd, scbid);
 1377                next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
 1378                scb = ahd_lookup_scb(ahd, scbid);
 1379                if (scb == NULL) {
 1380                        printf("%s: Warning - Complete SCB %d invalid\n",
 1381                               ahd_name(ahd), scbid);
 1382                        continue;
 1383                }
 1384
 1385                ahd_complete_scb(ahd, scb);
 1386                scbid = next_scbid;
 1387        }
 1388        ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
 1389
 1390        /*
 1391         * Restore state.
 1392         */
 1393        ahd_set_scbptr(ahd, saved_scbptr);
 1394        ahd_restore_modes(ahd, saved_modes);
 1395        ahd->flags |= AHD_UPDATE_PEND_CMDS;
 1396}
 1397
 1398/*
 1399 * Determine if an SCB for a packetized transaction
 1400 * is active in a FIFO.
 1401 */
 1402static int
 1403ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
 1404{
 1405
 1406        /*
 1407         * The FIFO is only active for our transaction if
 1408         * the SCBPTR matches the SCB's ID and the firmware
 1409         * has installed a handler for the FIFO or we have
 1410         * a pending SAVEPTRS or CFG4DATA interrupt.
 1411         */
 1412        if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
 1413         || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
 1414          && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
 1415                return (0);
 1416
 1417        return (1);
 1418}
 1419
 1420/*
 1421 * Run a data fifo to completion for a transaction we know
 1422 * has completed across the SCSI bus (good status has been
 1423 * received).  We are already set to the correct FIFO mode
 1424 * on entry to this routine.
 1425 *
 1426 * This function attempts to operate exactly as the firmware
 1427 * would when running this FIFO.  Care must be taken to update
 1428 * this routine any time the firmware's FIFO algorithm is
 1429 * changed.
 1430 */
 1431static void
 1432ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
 1433{
 1434        u_int seqintsrc;
 1435
 1436        seqintsrc = ahd_inb(ahd, SEQINTSRC);
 1437        if ((seqintsrc & CFG4DATA) != 0) {
 1438                uint32_t datacnt;
 1439                uint32_t sgptr;
 1440
 1441                /*
 1442                 * Clear full residual flag.
 1443                 */
 1444                sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
 1445                ahd_outb(ahd, SCB_SGPTR, sgptr);
 1446
 1447                /*
 1448                 * Load datacnt and address.
 1449                 */
 1450                datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
 1451                if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
 1452                        sgptr |= LAST_SEG;
 1453                        ahd_outb(ahd, SG_STATE, 0);
 1454                } else
 1455                        ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 1456                ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
 1457                ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
 1458                ahd_outb(ahd, SG_CACHE_PRE, sgptr);
 1459                ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
 1460
 1461                /*
 1462                 * Initialize Residual Fields.
 1463                 */
 1464                ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
 1465                ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
 1466
 1467                /*
 1468                 * Mark the SCB as having a FIFO in use.
 1469                 */
 1470                ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 1471                         ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
 1472
 1473                /*
 1474                 * Install a "fake" handler for this FIFO.
 1475                 */
 1476                ahd_outw(ahd, LONGJMP_ADDR, 0);
 1477
 1478                /*
 1479                 * Notify the hardware that we have satisfied
 1480                 * this sequencer interrupt.
 1481                 */
 1482                ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
 1483        } else if ((seqintsrc & SAVEPTRS) != 0) {
 1484                uint32_t sgptr;
 1485                uint32_t resid;
 1486
 1487                if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
 1488                        /*
 1489                         * Snapshot Save Pointers.  All that
 1490                         * is necessary to clear the snapshot
 1491                         * is a CLRCHN.
 1492                         */
 1493                        goto clrchn;
 1494                }
 1495
 1496                /*
 1497                 * Disable S/G fetch so the DMA engine
 1498                 * is available to future users.
 1499                 */
 1500                if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
 1501                        ahd_outb(ahd, CCSGCTL, 0);
 1502                ahd_outb(ahd, SG_STATE, 0);
 1503
 1504                /*
 1505                 * Flush the data FIFO.  Strickly only
 1506                 * necessary for Rev A parts.
 1507                 */
 1508                ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
 1509
 1510                /*
 1511                 * Calculate residual.
 1512                 */
 1513                sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 1514                resid = ahd_inl(ahd, SHCNT);
 1515                resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
 1516                ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
 1517                if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
 1518                        /*
 1519                         * Must back up to the correct S/G element.
 1520                         * Typically this just means resetting our
 1521                         * low byte to the offset in the SG_CACHE,
 1522                         * but if we wrapped, we have to correct
 1523                         * the other bytes of the sgptr too.
 1524                         */
 1525                        if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
 1526                         && (sgptr & 0x80) == 0)
 1527                                sgptr -= 0x100;
 1528                        sgptr &= ~0xFF;
 1529                        sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
 1530                               & SG_ADDR_MASK;
 1531                        ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 1532                        ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
 1533                } else if ((resid & AHD_SG_LEN_MASK) == 0) {
 1534                        ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
 1535                                 sgptr | SG_LIST_NULL);
 1536                }
 1537                /*
 1538                 * Save Pointers.
 1539                 */
 1540                ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
 1541                ahd_outl(ahd, SCB_DATACNT, resid);
 1542                ahd_outl(ahd, SCB_SGPTR, sgptr);
 1543                ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
 1544                ahd_outb(ahd, SEQIMODE,
 1545                         ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
 1546                /*
 1547                 * If the data is to the SCSI bus, we are
 1548                 * done, otherwise wait for FIFOEMP.
 1549                 */
 1550                if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
 1551                        goto clrchn;
 1552        } else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
 1553                uint32_t sgptr;
 1554                uint64_t data_addr;
 1555                uint32_t data_len;
 1556                u_int         dfcntrl;
 1557
 1558                /*
 1559                 * Disable S/G fetch so the DMA engine
 1560                 * is available to future users.  We won't
 1561                 * be using the DMA engine to load segments.
 1562                 */
 1563                if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
 1564                        ahd_outb(ahd, CCSGCTL, 0);
 1565                        ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
 1566                }
 1567
 1568                /*
 1569                 * Wait for the DMA engine to notice that the
 1570                 * host transfer is enabled and that there is
 1571                 * space in the S/G FIFO for new segments before
 1572                 * loading more segments.
 1573                 */
 1574                if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
 1575                 && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
 1576
 1577                        /*
 1578                         * Determine the offset of the next S/G
 1579                         * element to load.
 1580                         */
 1581                        sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 1582                        sgptr &= SG_PTR_MASK;
 1583                        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 1584                                struct ahd_dma64_seg *sg;
 1585
 1586                                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 1587                                data_addr = sg->addr;
 1588                                data_len = sg->len;
 1589                                sgptr += sizeof(*sg);
 1590                        } else {
 1591                                struct        ahd_dma_seg *sg;
 1592
 1593                                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 1594                                data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
 1595                                data_addr <<= 8;
 1596                                data_addr |= sg->addr;
 1597                                data_len = sg->len;
 1598                                sgptr += sizeof(*sg);
 1599                        }
 1600
 1601                        /*
 1602                         * Update residual information.
 1603                         */
 1604                        ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
 1605                        ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 1606
 1607                        /*
 1608                         * Load the S/G.
 1609                         */
 1610                        if (data_len & AHD_DMA_LAST_SEG) {
 1611                                sgptr |= LAST_SEG;
 1612                                ahd_outb(ahd, SG_STATE, 0);
 1613                        }
 1614                        ahd_outq(ahd, HADDR, data_addr);
 1615                        ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
 1616                        ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
 1617
 1618                        /*
 1619                         * Advertise the segment to the hardware.
 1620                         */
 1621                        dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
 1622                        if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
 1623                                /*
 1624                                 * Use SCSIENWRDIS so that SCSIEN
 1625                                 * is never modified by this
 1626                                 * operation.
 1627                                 */
 1628                                dfcntrl |= SCSIENWRDIS;
 1629                        }
 1630                        ahd_outb(ahd, DFCNTRL, dfcntrl);
 1631                }
 1632        } else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
 1633
 1634                /*
 1635                 * Transfer completed to the end of SG list
 1636                 * and has flushed to the host.
 1637                 */
 1638                ahd_outb(ahd, SCB_SGPTR,
 1639                         ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
 1640                goto clrchn;
 1641        } else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
 1642clrchn:
 1643                /*
 1644                 * Clear any handler for this FIFO, decrement
 1645                 * the FIFO use count for the SCB, and release
 1646                 * the FIFO.
 1647                 */
 1648                ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 1649                ahd_outb(ahd, SCB_FIFO_USE_COUNT,
 1650                         ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
 1651                ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
 1652        }
 1653}
 1654
 1655/*
 1656 * Look for entries in the QoutFIFO that have completed.
 1657 * The valid_tag completion field indicates the validity
 1658 * of the entry - the valid value toggles each time through
 1659 * the queue. We use the sg_status field in the completion
 1660 * entry to avoid referencing the hscb if the completion
 1661 * occurred with no errors and no residual.  sg_status is
 1662 * a copy of the first byte (little endian) of the sgptr
 1663 * hscb field.
 1664 */
 1665static void
 1666ahd_run_qoutfifo(struct ahd_softc *ahd)
 1667{
 1668        struct ahd_completion *completion;
 1669        struct scb *scb;
 1670        u_int  scb_index;
 1671
 1672        if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
 1673                panic("ahd_run_qoutfifo recursion");
 1674        ahd->flags |= AHD_RUNNING_QOUTFIFO;
 1675        ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
 1676        for (;;) {
 1677                completion = &ahd->qoutfifo[ahd->qoutfifonext];
 1678
 1679                if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
 1680                        break;
 1681
 1682                scb_index = ahd_le16toh(completion->tag);
 1683                scb = ahd_lookup_scb(ahd, scb_index);
 1684                if (scb == NULL) {
 1685                        printf("%s: WARNING no command for scb %d "
 1686                               "(cmdcmplt)\nQOUTPOS = %d\n",
 1687                               ahd_name(ahd), scb_index,
 1688                               ahd->qoutfifonext);
 1689                        ahd_dump_card_state(ahd);
 1690                } else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
 1691                        ahd_handle_scb_status(ahd, scb);
 1692                } else {
 1693                        ahd_done(ahd, scb);
 1694                }
 1695
 1696                ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
 1697                if (ahd->qoutfifonext == 0)
 1698                        ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
 1699        }
 1700        ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
 1701}
 1702
 1703/************************* Interrupt Handling *********************************/
 1704static void
 1705ahd_handle_hwerrint(struct ahd_softc *ahd)
 1706{
 1707        /*
 1708         * Some catastrophic hardware error has occurred.
 1709         * Print it for the user and disable the controller.
 1710         */
 1711        int i;
 1712        int error;
 1713
 1714        error = ahd_inb(ahd, ERROR);
 1715        for (i = 0; i < num_errors; i++) {
 1716                if ((error & ahd_hard_errors[i].errno) != 0)
 1717                        printf("%s: hwerrint, %s\n",
 1718                               ahd_name(ahd), ahd_hard_errors[i].errmesg);
 1719        }
 1720
 1721        ahd_dump_card_state(ahd);
 1722        panic("BRKADRINT");
 1723
 1724        /* Tell everyone that this HBA is no longer available */
 1725        ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 1726                       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 1727                       CAM_NO_HBA);
 1728
 1729        /* Tell the system that this controller has gone away. */
 1730        ahd_free(ahd);
 1731}
 1732
 1733#ifdef AHD_DEBUG
 1734static void
 1735ahd_dump_sglist(struct scb *scb)
 1736{
 1737        int i;
 1738
 1739        if (scb->sg_count > 0) {
 1740                if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
 1741                        struct ahd_dma64_seg *sg_list;
 1742
 1743                        sg_list = (struct ahd_dma64_seg*)scb->sg_list;
 1744                        for (i = 0; i < scb->sg_count; i++) {
 1745                                uint64_t addr;
 1746                                uint32_t len;
 1747
 1748                                addr = ahd_le64toh(sg_list[i].addr);
 1749                                len = ahd_le32toh(sg_list[i].len);
 1750                                printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
 1751                                       i,
 1752                                       (uint32_t)((addr >> 32) & 0xFFFFFFFF),
 1753                                       (uint32_t)(addr & 0xFFFFFFFF),
 1754                                       sg_list[i].len & AHD_SG_LEN_MASK,
 1755                                       (sg_list[i].len & AHD_DMA_LAST_SEG)
 1756                                     ? " Last" : "");
 1757                        }
 1758                } else {
 1759                        struct ahd_dma_seg *sg_list;
 1760
 1761                        sg_list = (struct ahd_dma_seg*)scb->sg_list;
 1762                        for (i = 0; i < scb->sg_count; i++) {
 1763                                uint32_t len;
 1764
 1765                                len = ahd_le32toh(sg_list[i].len);
 1766                                printf("sg[%d] - Addr 0x%x%x : Length %d%s\n",
 1767                                       i,
 1768                                       (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
 1769                                       ahd_le32toh(sg_list[i].addr),
 1770                                       len & AHD_SG_LEN_MASK,
 1771                                       len & AHD_DMA_LAST_SEG ? " Last" : "");
 1772                        }
 1773                }
 1774        }
 1775}
 1776#endif  /*  AHD_DEBUG  */
 1777
 1778static void
 1779ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
 1780{
 1781        u_int seqintcode;
 1782
 1783        /*
 1784         * Save the sequencer interrupt code and clear the SEQINT
 1785         * bit. We will unpause the sequencer, if appropriate,
 1786         * after servicing the request.
 1787         */
 1788        seqintcode = ahd_inb(ahd, SEQINTCODE);
 1789        ahd_outb(ahd, CLRINT, CLRSEQINT);
 1790        if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
 1791                /*
 1792                 * Unpause the sequencer and let it clear
 1793                 * SEQINT by writing NO_SEQINT to it.  This
 1794                 * will cause the sequencer to be paused again,
 1795                 * which is the expected state of this routine.
 1796                 */
 1797                ahd_unpause(ahd);
 1798                while (!ahd_is_paused(ahd))
 1799                        ;
 1800                ahd_outb(ahd, CLRINT, CLRSEQINT);
 1801        }
 1802        ahd_update_modes(ahd);
 1803#ifdef AHD_DEBUG
 1804        if ((ahd_debug & AHD_SHOW_MISC) != 0)
 1805                printf("%s: Handle Seqint Called for code %d\n",
 1806                       ahd_name(ahd), seqintcode);
 1807#endif
 1808        switch (seqintcode) {
 1809        case ENTERING_NONPACK:
 1810        {
 1811                struct        scb *scb;
 1812                u_int        scbid;
 1813
 1814                AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 1815                                 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 1816                scbid = ahd_get_scbptr(ahd);
 1817                scb = ahd_lookup_scb(ahd, scbid);
 1818                if (scb == NULL) {
 1819                        /*
 1820                         * Somehow need to know if this
 1821                         * is from a selection or reselection.
 1822                         * From that, we can determine target
 1823                         * ID so we at least have an I_T nexus.
 1824                         */
 1825                } else {
 1826                        ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 1827                        ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
 1828                        ahd_outb(ahd, SEQ_FLAGS, 0x0);
 1829                }
 1830                if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
 1831                 && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
 1832                        /*
 1833                         * Phase change after read stream with
 1834                         * CRC error with P0 asserted on last
 1835                         * packet.
 1836                         */
 1837#ifdef AHD_DEBUG
 1838                        if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 1839                                printf("%s: Assuming LQIPHASE_NLQ with "
 1840                                       "P0 assertion\n", ahd_name(ahd));
 1841#endif
 1842                }
 1843#ifdef AHD_DEBUG
 1844                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 1845                        printf("%s: Entering NONPACK\n", ahd_name(ahd));
 1846#endif
 1847                break;
 1848        }
 1849        case INVALID_SEQINT:
 1850                printf("%s: Invalid Sequencer interrupt occurred, "
 1851                       "resetting channel.\n",
 1852                       ahd_name(ahd));
 1853#ifdef AHD_DEBUG
 1854                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
 1855                        ahd_dump_card_state(ahd);
 1856#endif
 1857                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 1858                break;
 1859        case STATUS_OVERRUN:
 1860        {
 1861                struct        scb *scb;
 1862                u_int        scbid;
 1863
 1864                scbid = ahd_get_scbptr(ahd);
 1865                scb = ahd_lookup_scb(ahd, scbid);
 1866                if (scb != NULL)
 1867                        ahd_print_path(ahd, scb);
 1868                else
 1869                        printf("%s: ", ahd_name(ahd));
 1870                printf("SCB %d Packetized Status Overrun", scbid);
 1871                ahd_dump_card_state(ahd);
 1872                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 1873                break;
 1874        }
 1875        case CFG4ISTAT_INTR:
 1876        {
 1877                struct        scb *scb;
 1878                u_int        scbid;
 1879
 1880                scbid = ahd_get_scbptr(ahd);
 1881                scb = ahd_lookup_scb(ahd, scbid);
 1882                if (scb == NULL) {
 1883                        ahd_dump_card_state(ahd);
 1884                        printf("CFG4ISTAT: Free SCB %d referenced", scbid);
 1885                        panic("For safety");
 1886                }
 1887                ahd_outq(ahd, HADDR, scb->sense_busaddr);
 1888                ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
 1889                ahd_outb(ahd, HCNT + 2, 0);
 1890                ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
 1891                ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
 1892                break;
 1893        }
 1894        case ILLEGAL_PHASE:
 1895        {
 1896                u_int bus_phase;
 1897
 1898                bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
 1899                printf("%s: ILLEGAL_PHASE 0x%x\n",
 1900                       ahd_name(ahd), bus_phase);
 1901
 1902                switch (bus_phase) {
 1903                case P_DATAOUT:
 1904                case P_DATAIN:
 1905                case P_DATAOUT_DT:
 1906                case P_DATAIN_DT:
 1907                case P_MESGOUT:
 1908                case P_STATUS:
 1909                case P_MESGIN:
 1910                        ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 1911                        printf("%s: Issued Bus Reset.\n", ahd_name(ahd));
 1912                        break;
 1913                case P_COMMAND:
 1914                {
 1915                        struct        ahd_devinfo devinfo;
 1916                        struct        scb *scb;
 1917                        struct        ahd_initiator_tinfo *targ_info;
 1918                        struct        ahd_tmode_tstate *tstate;
 1919                        struct        ahd_transinfo *tinfo;
 1920                        u_int        scbid;
 1921
 1922                        /*
 1923                         * If a target takes us into the command phase
 1924                         * assume that it has been externally reset and
 1925                         * has thus lost our previous packetized negotiation
 1926                         * agreement.  Since we have not sent an identify
 1927                         * message and may not have fully qualified the
 1928                         * connection, we change our command to TUR, assert
 1929                         * ATN and ABORT the task when we go to message in
 1930                         * phase.  The OSM will see the REQUEUE_REQUEST
 1931                         * status and retry the command.
 1932                         */
 1933                        scbid = ahd_get_scbptr(ahd);
 1934                        scb = ahd_lookup_scb(ahd, scbid);
 1935                        if (scb == NULL) {
 1936                                printf("Invalid phase with no valid SCB.  "
 1937                                       "Resetting bus.\n");
 1938                                ahd_reset_channel(ahd, 'A',
 1939                                                  /*Initiate Reset*/TRUE);
 1940                                break;
 1941                        }
 1942                        ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
 1943                                            SCB_GET_TARGET(ahd, scb),
 1944                                            SCB_GET_LUN(scb),
 1945                                            SCB_GET_CHANNEL(ahd, scb),
 1946                                            ROLE_INITIATOR);
 1947                        targ_info = ahd_fetch_transinfo(ahd,
 1948                                                        devinfo.channel,
 1949                                                        devinfo.our_scsiid,
 1950                                                        devinfo.target,
 1951                                                        &tstate);
 1952                        tinfo = &targ_info->curr;
 1953                        ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
 1954                                      AHD_TRANS_ACTIVE, /*paused*/TRUE);
 1955                        ahd_set_syncrate(ahd, &devinfo, /*period*/0,
 1956                                         /*offset*/0, /*ppr_options*/0,
 1957                                         AHD_TRANS_ACTIVE, /*paused*/TRUE);
 1958                        /* Hand-craft TUR command */
 1959                        ahd_outb(ahd, SCB_CDB_STORE, 0);
 1960                        ahd_outb(ahd, SCB_CDB_STORE+1, 0);
 1961                        ahd_outb(ahd, SCB_CDB_STORE+2, 0);
 1962                        ahd_outb(ahd, SCB_CDB_STORE+3, 0);
 1963                        ahd_outb(ahd, SCB_CDB_STORE+4, 0);
 1964                        ahd_outb(ahd, SCB_CDB_STORE+5, 0);
 1965                        ahd_outb(ahd, SCB_CDB_LEN, 6);
 1966                        scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
 1967                        scb->hscb->control |= MK_MESSAGE;
 1968                        ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
 1969                        ahd_outb(ahd, MSG_OUT, HOST_MSG);
 1970                        ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
 1971                        /*
 1972                         * The lun is 0, regardless of the SCB's lun
 1973                         * as we have not sent an identify message.
 1974                         */
 1975                        ahd_outb(ahd, SAVED_LUN, 0);
 1976                        ahd_outb(ahd, SEQ_FLAGS, 0);
 1977                        ahd_assert_atn(ahd);
 1978                        scb->flags &= ~SCB_PACKETIZED;
 1979                        scb->flags |= SCB_ABORT|SCB_EXTERNAL_RESET;
 1980                        ahd_freeze_devq(ahd, scb);
 1981                        ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
 1982                        ahd_freeze_scb(scb);
 1983
 1984                        /* Notify XPT */
 1985                        ahd_send_async(ahd, devinfo.channel, devinfo.target,
 1986                                       CAM_LUN_WILDCARD, AC_SENT_BDR);
 1987
 1988                        /*
 1989                         * Allow the sequencer to continue with
 1990                         * non-pack processing.
 1991                         */
 1992                        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 1993                        ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
 1994                        if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
 1995                                ahd_outb(ahd, CLRLQOINT1, 0);
 1996                        }
 1997#ifdef AHD_DEBUG
 1998                        if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
 1999                                ahd_print_path(ahd, scb);
 2000                                printf("Unexpected command phase from "
 2001                                       "packetized target\n");
 2002                        }
 2003#endif
 2004                        break;
 2005                }
 2006                }
 2007                break;
 2008        }
 2009        case CFG4OVERRUN:
 2010        {
 2011                struct        scb *scb;
 2012                u_int        scb_index;
 2013                
 2014#ifdef AHD_DEBUG
 2015                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
 2016                        printf("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
 2017                               ahd_inb(ahd, MODE_PTR));
 2018                }
 2019#endif
 2020                scb_index = ahd_get_scbptr(ahd);
 2021                scb = ahd_lookup_scb(ahd, scb_index);
 2022                if (scb == NULL) {
 2023                        /*
 2024                         * Attempt to transfer to an SCB that is
 2025                         * not outstanding.
 2026                         */
 2027                        ahd_assert_atn(ahd);
 2028                        ahd_outb(ahd, MSG_OUT, HOST_MSG);
 2029                        ahd->msgout_buf[0] = MSG_ABORT_TASK;
 2030                        ahd->msgout_len = 1;
 2031                        ahd->msgout_index = 0;
 2032                        ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 2033                        /*
 2034                         * Clear status received flag to prevent any
 2035                         * attempt to complete this bogus SCB.
 2036                         */
 2037                        ahd_outb(ahd, SCB_CONTROL,
 2038                                 ahd_inb_scbram(ahd, SCB_CONTROL)
 2039                                 & ~STATUS_RCVD);
 2040                }
 2041                break;
 2042        }
 2043        case DUMP_CARD_STATE:
 2044        {
 2045                ahd_dump_card_state(ahd);
 2046                break;
 2047        }
 2048        case PDATA_REINIT:
 2049        {
 2050#ifdef AHD_DEBUG
 2051                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
 2052                        printf("%s: PDATA_REINIT - DFCNTRL = 0x%x "
 2053                               "SG_CACHE_SHADOW = 0x%x\n",
 2054                               ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
 2055                               ahd_inb(ahd, SG_CACHE_SHADOW));
 2056                }
 2057#endif
 2058                ahd_reinitialize_dataptrs(ahd);
 2059                break;
 2060        }
 2061        case HOST_MSG_LOOP:
 2062        {
 2063                struct ahd_devinfo devinfo;
 2064
 2065                /*
 2066                 * The sequencer has encountered a message phase
 2067                 * that requires host assistance for completion.
 2068                 * While handling the message phase(s), we will be
 2069                 * notified by the sequencer after each byte is
 2070                 * transfered so we can track bus phase changes.
 2071                 *
 2072                 * If this is the first time we've seen a HOST_MSG_LOOP
 2073                 * interrupt, initialize the state of the host message
 2074                 * loop.
 2075                 */
 2076                ahd_fetch_devinfo(ahd, &devinfo);
 2077                if (ahd->msg_type == MSG_TYPE_NONE) {
 2078                        struct scb *scb;
 2079                        u_int scb_index;
 2080                        u_int bus_phase;
 2081
 2082                        bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
 2083                        if (bus_phase != P_MESGIN
 2084                         && bus_phase != P_MESGOUT) {
 2085                                printf("ahd_intr: HOST_MSG_LOOP bad "
 2086                                       "phase 0x%x\n", bus_phase);
 2087                                /*
 2088                                 * Probably transitioned to bus free before
 2089                                 * we got here.  Just punt the message.
 2090                                 */
 2091                                ahd_dump_card_state(ahd);
 2092                                ahd_clear_intstat(ahd);
 2093                                ahd_restart(ahd);
 2094                                return;
 2095                        }
 2096
 2097                        scb_index = ahd_get_scbptr(ahd);
 2098                        scb = ahd_lookup_scb(ahd, scb_index);
 2099                        if (devinfo.role == ROLE_INITIATOR) {
 2100                                if (bus_phase == P_MESGOUT)
 2101                                        ahd_setup_initiator_msgout(ahd,
 2102                                                                   &devinfo,
 2103                                                                   scb);
 2104                                else {
 2105                                        ahd->msg_type =
 2106                                            MSG_TYPE_INITIATOR_MSGIN;
 2107                                        ahd->msgin_index = 0;
 2108                                }
 2109                        }
 2110#ifdef AHD_TARGET_MODE
 2111                        else {
 2112                                if (bus_phase == P_MESGOUT) {
 2113                                        ahd->msg_type =
 2114                                            MSG_TYPE_TARGET_MSGOUT;
 2115                                        ahd->msgin_index = 0;
 2116                                }
 2117                                else 
 2118                                        ahd_setup_target_msgin(ahd,
 2119                                                               &devinfo,
 2120                                                               scb);
 2121                        }
 2122#endif
 2123                }
 2124
 2125                ahd_handle_message_phase(ahd);
 2126                break;
 2127        }
 2128        case NO_MATCH:
 2129        {
 2130                /* Ensure we don't leave the selection hardware on */
 2131                AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 2132                ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
 2133
 2134                printf("%s:%c:%d: no active SCB for reconnecting "
 2135                       "target - issuing BUS DEVICE RESET\n",
 2136                       ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
 2137                printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
 2138                       "REG0 == 0x%x ACCUM = 0x%x\n",
 2139                       ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
 2140                       ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
 2141                printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
 2142                       "SINDEX == 0x%x\n",
 2143                       ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
 2144                       ahd_find_busy_tcl(ahd,
 2145                                         BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
 2146                                                   ahd_inb(ahd, SAVED_LUN))),
 2147                       ahd_inw(ahd, SINDEX));
 2148                printf("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
 2149                       "SCB_CONTROL == 0x%x\n",
 2150                       ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
 2151                       ahd_inb_scbram(ahd, SCB_LUN),
 2152                       ahd_inb_scbram(ahd, SCB_CONTROL));
 2153                printf("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
 2154                       ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
 2155                printf("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
 2156                printf("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
 2157                ahd_dump_card_state(ahd);
 2158                ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
 2159                ahd->msgout_len = 1;
 2160                ahd->msgout_index = 0;
 2161                ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 2162                ahd_outb(ahd, MSG_OUT, HOST_MSG);
 2163                ahd_assert_atn(ahd);
 2164                break;
 2165        }
 2166        case PROTO_VIOLATION:
 2167        {
 2168                ahd_handle_proto_violation(ahd);
 2169                break;
 2170        }
 2171        case IGN_WIDE_RES:
 2172        {
 2173                struct ahd_devinfo devinfo;
 2174
 2175                ahd_fetch_devinfo(ahd, &devinfo);
 2176                ahd_handle_ign_wide_residue(ahd, &devinfo);
 2177                break;
 2178        }
 2179        case BAD_PHASE:
 2180        {
 2181                u_int lastphase;
 2182
 2183                lastphase = ahd_inb(ahd, LASTPHASE);
 2184                printf("%s:%c:%d: unknown scsi bus phase %x, "
 2185                       "lastphase = 0x%x.  Attempting to continue\n",
 2186                       ahd_name(ahd), 'A',
 2187                       SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
 2188                       lastphase, ahd_inb(ahd, SCSISIGI));
 2189                break;
 2190        }
 2191        case MISSED_BUSFREE:
 2192        {
 2193                u_int lastphase;
 2194
 2195                lastphase = ahd_inb(ahd, LASTPHASE);
 2196                printf("%s:%c:%d: Missed busfree. "
 2197                       "Lastphase = 0x%x, Curphase = 0x%x\n",
 2198                       ahd_name(ahd), 'A',
 2199                       SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
 2200                       lastphase, ahd_inb(ahd, SCSISIGI));
 2201                ahd_restart(ahd);
 2202                return;
 2203        }
 2204        case DATA_OVERRUN:
 2205        {
 2206                /*
 2207                 * When the sequencer detects an overrun, it
 2208                 * places the controller in "BITBUCKET" mode
 2209                 * and allows the target to complete its transfer.
 2210                 * Unfortunately, none of the counters get updated
 2211                 * when the controller is in this mode, so we have
 2212                 * no way of knowing how large the overrun was.
 2213                 */
 2214                struct        scb *scb;
 2215                u_int        scbindex;
 2216#ifdef AHD_DEBUG
 2217                u_int        lastphase;
 2218#endif
 2219
 2220                scbindex = ahd_get_scbptr(ahd);
 2221                scb = ahd_lookup_scb(ahd, scbindex);
 2222#ifdef AHD_DEBUG
 2223                lastphase = ahd_inb(ahd, LASTPHASE);
 2224                if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
 2225                        ahd_print_path(ahd, scb);
 2226                        printf("data overrun detected %s.  Tag == 0x%x.\n",
 2227                               ahd_lookup_phase_entry(lastphase)->phasemsg,
 2228                               SCB_GET_TAG(scb));
 2229                        ahd_print_path(ahd, scb);
 2230                        printf("%s seen Data Phase.  Length = %ld.  "
 2231                               "NumSGs = %d.\n",
 2232                               ahd_inb(ahd, SEQ_FLAGS) & DPHASE
 2233                               ? "Have" : "Haven't",
 2234                               ahd_get_transfer_length(scb), scb->sg_count);
 2235                        ahd_dump_sglist(scb);
 2236                }
 2237#endif
 2238
 2239                /*
 2240                 * Set this and it will take effect when the
 2241                 * target does a command complete.
 2242                 */
 2243                ahd_freeze_devq(ahd, scb);
 2244                ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
 2245                ahd_freeze_scb(scb);
 2246                break;
 2247        }
 2248        case MKMSG_FAILED:
 2249        {
 2250                struct ahd_devinfo devinfo;
 2251                struct scb *scb;
 2252                u_int scbid;
 2253
 2254                ahd_fetch_devinfo(ahd, &devinfo);
 2255                printf("%s:%c:%d:%d: Attempt to issue message failed\n",
 2256                       ahd_name(ahd), devinfo.channel, devinfo.target,
 2257                       devinfo.lun);
 2258                scbid = ahd_get_scbptr(ahd);
 2259                scb = ahd_lookup_scb(ahd, scbid);
 2260                if (scb != NULL
 2261                 && (scb->flags & SCB_RECOVERY_SCB) != 0)
 2262                        /*
 2263                         * Ensure that we didn't put a second instance of this
 2264                         * SCB into the QINFIFO.
 2265                         */
 2266                        ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
 2267                                           SCB_GET_CHANNEL(ahd, scb),
 2268                                           SCB_GET_LUN(scb), SCB_GET_TAG(scb),
 2269                                           ROLE_INITIATOR, /*status*/0,
 2270                                           SEARCH_REMOVE);
 2271                ahd_outb(ahd, SCB_CONTROL,
 2272                         ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
 2273                break;
 2274        }
 2275        case TASKMGMT_FUNC_COMPLETE:
 2276        {
 2277                u_int        scbid;
 2278                struct        scb *scb;
 2279
 2280                scbid = ahd_get_scbptr(ahd);
 2281                scb = ahd_lookup_scb(ahd, scbid);
 2282                if (scb != NULL) {
 2283                        u_int           lun;
 2284                        u_int           tag;
 2285                        cam_status error;
 2286
 2287                        ahd_print_path(ahd, scb);
 2288                        printf("Task Management Func 0x%x Complete\n",
 2289                               scb->hscb->task_management);
 2290                        lun = CAM_LUN_WILDCARD;
 2291                        tag = SCB_LIST_NULL;
 2292
 2293                        switch (scb->hscb->task_management) {
 2294                        case SIU_TASKMGMT_ABORT_TASK:
 2295                                tag = SCB_GET_TAG(scb);
 2296                        case SIU_TASKMGMT_ABORT_TASK_SET:
 2297                        case SIU_TASKMGMT_CLEAR_TASK_SET:
 2298                                lun = scb->hscb->lun;
 2299                                error = CAM_REQ_ABORTED;
 2300                                ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
 2301                                               'A', lun, tag, ROLE_INITIATOR,
 2302                                               error);
 2303                                break;
 2304                        case SIU_TASKMGMT_LUN_RESET:
 2305                                lun = scb->hscb->lun;
 2306                        case SIU_TASKMGMT_TARGET_RESET:
 2307                        {
 2308                                struct ahd_devinfo devinfo;
 2309
 2310                                ahd_scb_devinfo(ahd, &devinfo, scb);
 2311                                error = CAM_BDR_SENT;
 2312                                ahd_handle_devreset(ahd, &devinfo, lun,
 2313                                                    CAM_BDR_SENT,
 2314                                                    lun != CAM_LUN_WILDCARD
 2315                                                    ? "Lun Reset"
 2316                                                    : "Target Reset",
 2317                                                    /*verbose_level*/0);
 2318                                break;
 2319                        }
 2320                        default:
 2321                                panic("Unexpected TaskMgmt Func\n");
 2322                                break;
 2323                        }
 2324                }
 2325                break;
 2326        }
 2327        case TASKMGMT_CMD_CMPLT_OKAY:
 2328        {
 2329                u_int        scbid;
 2330                struct        scb *scb;
 2331
 2332                /*
 2333                 * An ABORT TASK TMF failed to be delivered before
 2334                 * the targeted command completed normally.
 2335                 */
 2336                scbid = ahd_get_scbptr(ahd);
 2337                scb = ahd_lookup_scb(ahd, scbid);
 2338                if (scb != NULL) {
 2339                        /*
 2340                         * Remove the second instance of this SCB from
 2341                         * the QINFIFO if it is still there.
 2342                         */
 2343                        ahd_print_path(ahd, scb);
 2344                        printf("SCB completes before TMF\n");
 2345                        /*
 2346                         * Handle losing the race.  Wait until any
 2347                         * current selection completes.  We will then
 2348                         * set the TMF back to zero in this SCB so that
 2349                         * the sequencer doesn't bother to issue another
 2350                         * sequencer interrupt for its completion.
 2351                         */
 2352                        while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
 2353                            && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
 2354                            && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
 2355                                ;
 2356                        ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
 2357                        ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
 2358                                           SCB_GET_CHANNEL(ahd, scb),  
 2359                                           SCB_GET_LUN(scb), SCB_GET_TAG(scb), 
 2360                                           ROLE_INITIATOR, /*status*/0,   
 2361                                           SEARCH_REMOVE);
 2362                }
 2363                break;
 2364        }
 2365        case TRACEPOINT0:
 2366        case TRACEPOINT1:
 2367        case TRACEPOINT2:
 2368        case TRACEPOINT3:
 2369                printf("%s: Tracepoint %d\n", ahd_name(ahd),
 2370                       seqintcode - TRACEPOINT0);
 2371                break;
 2372        case NO_SEQINT:
 2373                break;
 2374        case SAW_HWERR:
 2375                ahd_handle_hwerrint(ahd);
 2376                break;
 2377        default:
 2378                printf("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
 2379                       seqintcode);
 2380                break;
 2381        }
 2382        /*
 2383         *  The sequencer is paused immediately on
 2384         *  a SEQINT, so we should restart it when
 2385         *  we're done.
 2386         */
 2387        ahd_unpause(ahd);
 2388}
 2389
 2390static void
 2391ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
 2392{
 2393        struct scb        *scb;
 2394        u_int                 status0;
 2395        u_int                 status3;
 2396        u_int                 status;
 2397        u_int                 lqistat1;
 2398        u_int                 lqostat0;
 2399        u_int                 scbid;
 2400        u_int                 busfreetime;
 2401
 2402        ahd_update_modes(ahd);
 2403        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 2404
 2405        status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
 2406        status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
 2407        status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
 2408        lqistat1 = ahd_inb(ahd, LQISTAT1);
 2409        lqostat0 = ahd_inb(ahd, LQOSTAT0);
 2410        busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
 2411
 2412        /*
 2413         * Ignore external resets after a bus reset.
 2414         */
 2415        if (((status & SCSIRSTI) != 0) && (ahd->flags & AHD_BUS_RESET_ACTIVE)) {
 2416                ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
 2417                return;
 2418        }
 2419
 2420        /*
 2421         * Clear bus reset flag
 2422         */
 2423        ahd->flags &= ~AHD_BUS_RESET_ACTIVE;
 2424
 2425        if ((status0 & (SELDI|SELDO)) != 0) {
 2426                u_int simode0;
 2427
 2428                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 2429                simode0 = ahd_inb(ahd, SIMODE0);
 2430                status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
 2431                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 2432        }
 2433        scbid = ahd_get_scbptr(ahd);
 2434        scb = ahd_lookup_scb(ahd, scbid);
 2435        if (scb != NULL
 2436         && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
 2437                scb = NULL;
 2438
 2439        if ((status0 & IOERR) != 0) {
 2440                u_int now_lvd;
 2441
 2442                now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
 2443                printf("%s: Transceiver State Has Changed to %s mode\n",
 2444                       ahd_name(ahd), now_lvd ? "LVD" : "SE");
 2445                ahd_outb(ahd, CLRSINT0, CLRIOERR);
 2446                /*
 2447                 * A change in I/O mode is equivalent to a bus reset.
 2448                 */
 2449                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 2450                ahd_pause(ahd);
 2451                ahd_setup_iocell_workaround(ahd);
 2452                ahd_unpause(ahd);
 2453        } else if ((status0 & OVERRUN) != 0) {
 2454
 2455                printf("%s: SCSI offset overrun detected.  Resetting bus.\n",
 2456                       ahd_name(ahd));
 2457                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 2458        } else if ((status & SCSIRSTI) != 0) {
 2459
 2460                printf("%s: Someone reset channel A\n", ahd_name(ahd));
 2461                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
 2462        } else if ((status & SCSIPERR) != 0) {
 2463
 2464                /* Make sure the sequencer is in a safe location. */
 2465                ahd_clear_critical_section(ahd);
 2466
 2467                ahd_handle_transmission_error(ahd);
 2468        } else if (lqostat0 != 0) {
 2469
 2470                printf("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
 2471                ahd_outb(ahd, CLRLQOINT0, lqostat0);
 2472                if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
 2473                        ahd_outb(ahd, CLRLQOINT1, 0);
 2474        } else if ((status & SELTO) != 0) {
 2475                /* Stop the selection */
 2476                ahd_outb(ahd, SCSISEQ0, 0);
 2477
 2478                /* Make sure the sequencer is in a safe location. */
 2479                ahd_clear_critical_section(ahd);
 2480
 2481                /* No more pending messages */
 2482                ahd_clear_msg_state(ahd);
 2483
 2484                /* Clear interrupt state */
 2485                ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
 2486
 2487                /*
 2488                 * Although the driver does not care about the
 2489                 * 'Selection in Progress' status bit, the busy
 2490                 * LED does.  SELINGO is only cleared by a sucessfull
 2491                 * selection, so we must manually clear it to insure
 2492                 * the LED turns off just incase no future successful
 2493                 * selections occur (e.g. no devices on the bus).
 2494                 */
 2495                ahd_outb(ahd, CLRSINT0, CLRSELINGO);
 2496
 2497                scbid = ahd_inw(ahd, WAITING_TID_HEAD);
 2498                scb = ahd_lookup_scb(ahd, scbid);
 2499                if (scb == NULL) {
 2500                        printf("%s: ahd_intr - referenced scb not "
 2501                               "valid during SELTO scb(0x%x)\n",
 2502                               ahd_name(ahd), scbid);
 2503                        ahd_dump_card_state(ahd);
 2504                } else {
 2505                        struct ahd_devinfo devinfo;
 2506#ifdef AHD_DEBUG
 2507                        if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
 2508                                ahd_print_path(ahd, scb);
 2509                                printf("Saw Selection Timeout for SCB 0x%x\n",
 2510                                       scbid);
 2511                        }
 2512#endif
 2513                        ahd_scb_devinfo(ahd, &devinfo, scb);
 2514                        ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
 2515                        ahd_freeze_devq(ahd, scb);
 2516
 2517                        /*
 2518                         * Cancel any pending transactions on the device
 2519                         * now that it seems to be missing.  This will
 2520                         * also revert us to async/narrow transfers until
 2521                         * we can renegotiate with the device.
 2522                         */
 2523                        ahd_handle_devreset(ahd, &devinfo,
 2524                                            CAM_LUN_WILDCARD,
 2525                                            CAM_SEL_TIMEOUT,
 2526                                            "Selection Timeout",
 2527                                            /*verbose_level*/1);
 2528                }
 2529                ahd_outb(ahd, CLRINT, CLRSCSIINT);
 2530                ahd_iocell_first_selection(ahd);
 2531                ahd_unpause(ahd);
 2532        } else if ((status0 & (SELDI|SELDO)) != 0) {
 2533
 2534                ahd_iocell_first_selection(ahd);
 2535                ahd_unpause(ahd);
 2536        } else if (status3 != 0) {
 2537                printf("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
 2538                       ahd_name(ahd), status3);
 2539                ahd_outb(ahd, CLRSINT3, status3);
 2540        } else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
 2541
 2542                /* Make sure the sequencer is in a safe location. */
 2543                ahd_clear_critical_section(ahd);
 2544
 2545                ahd_handle_lqiphase_error(ahd, lqistat1);
 2546        } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
 2547                /*
 2548                 * This status can be delayed during some
 2549                 * streaming operations.  The SCSIPHASE
 2550                 * handler has already dealt with this case
 2551                 * so just clear the error.
 2552                 */
 2553                ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
 2554        } else if ((status & BUSFREE) != 0
 2555                || (lqistat1 & LQOBUSFREE) != 0) {
 2556                u_int lqostat1;
 2557                int   restart;
 2558                int   clear_fifo;
 2559                int   packetized;
 2560                u_int mode;
 2561
 2562                /*
 2563                 * Clear our selection hardware as soon as possible.
 2564                 * We may have an entry in the waiting Q for this target,
 2565                 * that is affected by this busfree and we don't want to
 2566                 * go about selecting the target while we handle the event.
 2567                 */
 2568                ahd_outb(ahd, SCSISEQ0, 0);
 2569
 2570                /* Make sure the sequencer is in a safe location. */
 2571                ahd_clear_critical_section(ahd);
 2572
 2573                /*
 2574                 * Determine what we were up to at the time of
 2575                 * the busfree.
 2576                 */
 2577                mode = AHD_MODE_SCSI;
 2578                busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
 2579                lqostat1 = ahd_inb(ahd, LQOSTAT1);
 2580                switch (busfreetime) {
 2581                case BUSFREE_DFF0:
 2582                case BUSFREE_DFF1:
 2583                {
 2584                        mode = busfreetime == BUSFREE_DFF0
 2585                             ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
 2586                        ahd_set_modes(ahd, mode, mode);
 2587                        scbid = ahd_get_scbptr(ahd);
 2588                        scb = ahd_lookup_scb(ahd, scbid);
 2589                        if (scb == NULL) {
 2590                                printf("%s: Invalid SCB %d in DFF%d "
 2591                                       "during unexpected busfree\n",
 2592                                       ahd_name(ahd), scbid, mode);
 2593                                packetized = 0;
 2594                        } else
 2595                                packetized = (scb->flags & SCB_PACKETIZED) != 0;
 2596                        clear_fifo = 1;
 2597                        break;
 2598                }
 2599                case BUSFREE_LQO:
 2600                        clear_fifo = 0;
 2601                        packetized = 1;
 2602                        break;
 2603                default:
 2604                        clear_fifo = 0;
 2605                        packetized =  (lqostat1 & LQOBUSFREE) != 0;
 2606                        if (!packetized
 2607                         && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
 2608                         && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
 2609                         && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
 2610                          || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
 2611                                /*
 2612                                 * Assume packetized if we are not
 2613                                 * on the bus in a non-packetized
 2614                                 * capacity and any pending selection
 2615                                 * was a packetized selection.
 2616                                 */
 2617                                packetized = 1;
 2618                        break;
 2619                }
 2620
 2621#ifdef AHD_DEBUG
 2622                if ((ahd_debug & AHD_SHOW_MISC) != 0)
 2623                        printf("Saw Busfree.  Busfreetime = 0x%x.\n",
 2624                               busfreetime);
 2625#endif
 2626                /*
 2627                 * Busfrees that occur in non-packetized phases are
 2628                 * handled by the nonpkt_busfree handler.
 2629                 */
 2630                if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
 2631                        restart = ahd_handle_pkt_busfree(ahd, busfreetime);
 2632                } else {
 2633                        packetized = 0;
 2634                        restart = ahd_handle_nonpkt_busfree(ahd);
 2635                }
 2636                /*
 2637                 * Clear the busfree interrupt status.  The setting of
 2638                 * the interrupt is a pulse, so in a perfect world, we
 2639                 * would not need to muck with the ENBUSFREE logic.  This
 2640                 * would ensure that if the bus moves on to another
 2641                 * connection, busfree protection is still in force.  If
 2642                 * BUSFREEREV is broken, however, we must manually clear
 2643                 * the ENBUSFREE if the busfree occurred during a non-pack
 2644                 * connection so that we don't get false positives during
 2645                 * future, packetized, connections.
 2646                 */
 2647                ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
 2648                if (packetized == 0
 2649                 && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
 2650                        ahd_outb(ahd, SIMODE1,
 2651                                 ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
 2652
 2653                if (clear_fifo)
 2654                        ahd_clear_fifo(ahd, mode);
 2655
 2656                ahd_clear_msg_state(ahd);
 2657                ahd_outb(ahd, CLRINT, CLRSCSIINT);
 2658                if (restart) {
 2659                        ahd_restart(ahd);
 2660                } else {
 2661                        ahd_unpause(ahd);
 2662                }
 2663        } else {
 2664                printf("%s: Missing case in ahd_handle_scsiint. status = %x\n",
 2665                       ahd_name(ahd), status);
 2666                ahd_dump_card_state(ahd);
 2667                ahd_clear_intstat(ahd);
 2668                ahd_unpause(ahd);
 2669        }
 2670}
 2671
 2672static void
 2673ahd_handle_transmission_error(struct ahd_softc *ahd)
 2674{
 2675        struct        scb *scb;
 2676        u_int        scbid;
 2677        u_int        lqistat1;
 2678        u_int        lqistat2;
 2679        u_int        msg_out;
 2680        u_int        curphase;
 2681        u_int        lastphase;
 2682        u_int        perrdiag;
 2683        u_int        cur_col;
 2684        int        silent;
 2685
 2686        scb = NULL;
 2687        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 2688        lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
 2689        lqistat2 = ahd_inb(ahd, LQISTAT2);
 2690        if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
 2691         && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
 2692                u_int lqistate;
 2693
 2694                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 2695                lqistate = ahd_inb(ahd, LQISTATE);
 2696                if ((lqistate >= 0x1E && lqistate <= 0x24)
 2697                 || (lqistate == 0x29)) {
 2698#ifdef AHD_DEBUG
 2699                        if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
 2700                                printf("%s: NLQCRC found via LQISTATE\n",
 2701                                       ahd_name(ahd));
 2702                        }
 2703#endif
 2704                        lqistat1 |= LQICRCI_NLQ;
 2705                }
 2706                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 2707        }
 2708
 2709        ahd_outb(ahd, CLRLQIINT1, lqistat1);
 2710        lastphase = ahd_inb(ahd, LASTPHASE);
 2711        curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
 2712        perrdiag = ahd_inb(ahd, PERRDIAG);
 2713        msg_out = MSG_INITIATOR_DET_ERR;
 2714        ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
 2715        
 2716        /*
 2717         * Try to find the SCB associated with this error.
 2718         */
 2719        silent = FALSE;
 2720        if (lqistat1 == 0
 2721         || (lqistat1 & LQICRCI_NLQ) != 0) {
 2722                 if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
 2723                        ahd_set_active_fifo(ahd);
 2724                scbid = ahd_get_scbptr(ahd);
 2725                scb = ahd_lookup_scb(ahd, scbid);
 2726                if (scb != NULL && SCB_IS_SILENT(scb))
 2727                        silent = TRUE;
 2728        }
 2729
 2730        cur_col = 0;
 2731        if (silent == FALSE) {
 2732                printf("%s: Transmission error detected\n", ahd_name(ahd));
 2733                ahd_lqistat1_print(lqistat1, &cur_col, 50);
 2734                ahd_lastphase_print(lastphase, &cur_col, 50);
 2735                ahd_scsisigi_print(curphase, &cur_col, 50);
 2736                ahd_perrdiag_print(perrdiag, &cur_col, 50);
 2737                printf("\n");
 2738                ahd_dump_card_state(ahd);
 2739        }
 2740
 2741        if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
 2742                if (silent == FALSE) {
 2743                        printf("%s: Gross protocol error during incoming "
 2744                               "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
 2745                               ahd_name(ahd), lqistat1);
 2746                }
 2747                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 2748                return;
 2749        } else if ((lqistat1 & LQICRCI_LQ) != 0) {
 2750                /*
 2751                 * A CRC error has been detected on an incoming LQ.
 2752                 * The bus is currently hung on the last ACK.
 2753                 * Hit LQIRETRY to release the last ack, and
 2754                 * wait for the sequencer to determine that ATNO
 2755                 * is asserted while in message out to take us
 2756                 * to our host message loop.  No NONPACKREQ or
 2757                 * LQIPHASE type errors will occur in this
 2758                 * scenario.  After this first LQIRETRY, the LQI
 2759                 * manager will be in ISELO where it will
 2760                 * happily sit until another packet phase begins.
 2761                 * Unexpected bus free detection is enabled
 2762                 * through any phases that occur after we release
 2763                 * this last ack until the LQI manager sees a
 2764                 * packet phase.  This implies we may have to
 2765                 * ignore a perfectly valid "unexected busfree"
 2766                 * after our "initiator detected error" message is
 2767                 * sent.  A busfree is the expected response after
 2768                 * we tell the target that it's L_Q was corrupted.
 2769                 * (SPI4R09 10.7.3.3.3)
 2770                 */
 2771                ahd_outb(ahd, LQCTL2, LQIRETRY);
 2772                printf("LQIRetry for LQICRCI_LQ to release ACK\n");
 2773        } else if ((lqistat1 & LQICRCI_NLQ) != 0) {
 2774                /*
 2775                 * We detected a CRC error in a NON-LQ packet.
 2776                 * The hardware has varying behavior in this situation
 2777                 * depending on whether this packet was part of a
 2778                 * stream or not.
 2779                 *
 2780                 * PKT by PKT mode:
 2781                 * The hardware has already acked the complete packet.
 2782                 * If the target honors our outstanding ATN condition,
 2783                 * we should be (or soon will be) in MSGOUT phase.
 2784                 * This will trigger the LQIPHASE_LQ status bit as the
 2785                 * hardware was expecting another LQ.  Unexpected
 2786                 * busfree detection is enabled.  Once LQIPHASE_LQ is
 2787                 * true (first entry into host message loop is much
 2788                 * the same), we must clear LQIPHASE_LQ and hit
 2789                 * LQIRETRY so the hardware is ready to handle
 2790                 * a future LQ.  NONPACKREQ will not be asserted again
 2791                 * once we hit LQIRETRY until another packet is
 2792                 * processed.  The target may either go busfree
 2793                 * or start another packet in response to our message.
 2794                 *
 2795                 * Read Streaming P0 asserted:
 2796                 * If we raise ATN and the target completes the entire
 2797                 * stream (P0 asserted during the last packet), the
 2798                 * hardware will ack all data and return to the ISTART
 2799                 * state.  When the target reponds to our ATN condition,
 2800                 * LQIPHASE_LQ will be asserted.  We should respond to
 2801                 * this with an LQIRETRY to prepare for any future
 2802                 * packets.  NONPACKREQ will not be asserted again
 2803                 * once we hit LQIRETRY until another packet is
 2804                 * processed.  The target may either go busfree or
 2805                 * start another packet in response to our message.
 2806                 * Busfree detection is enabled.
 2807                 *
 2808                 * Read Streaming P0 not asserted:
 2809                 * If we raise ATN and the target transitions to
 2810                 * MSGOUT in or after a packet where P0 is not
 2811                 * asserted, the hardware will assert LQIPHASE_NLQ.
 2812                 * We should respond to the LQIPHASE_NLQ with an
 2813                 * LQIRETRY.  Should the target stay in a non-pkt
 2814                 * phase after we send our message, the hardware
 2815                 * will assert LQIPHASE_LQ.  Recovery is then just as
 2816                 * listed above for the read streaming with P0 asserted.
 2817                 * Busfree detection is enabled.
 2818                 */
 2819                if (silent == FALSE)
 2820                        printf("LQICRC_NLQ\n");
 2821                if (scb == NULL) {
 2822                        printf("%s: No SCB valid for LQICRC_NLQ.  "
 2823                               "Resetting bus\n", ahd_name(ahd));
 2824                        ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 2825                        return;
 2826                }
 2827        } else if ((lqistat1 & LQIBADLQI) != 0) {
 2828                printf("Need to handle BADLQI!\n");
 2829                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 2830                return;
 2831        } else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
 2832                if ((curphase & ~P_DATAIN_DT) != 0) {
 2833                        /* Ack the byte.  So we can continue. */
 2834                        if (silent == FALSE)
 2835                                printf("Acking %s to clear perror\n",
 2836                                    ahd_lookup_phase_entry(curphase)->phasemsg);
 2837                        ahd_inb(ahd, SCSIDAT);
 2838                }
 2839        
 2840                if (curphase == P_MESGIN)
 2841                        msg_out = MSG_PARITY_ERROR;
 2842        }
 2843
 2844        /*
 2845         * We've set the hardware to assert ATN if we 
 2846         * get a parity error on "in" phases, so all we
 2847         * need to do is stuff the message buffer with
 2848         * the appropriate message.  "In" phases have set
 2849         * mesg_out to something other than MSG_NOP.
 2850         */
 2851        ahd->send_msg_perror = msg_out;
 2852        if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
 2853                scb->flags |= SCB_TRANSMISSION_ERROR;
 2854        ahd_outb(ahd, MSG_OUT, HOST_MSG);
 2855        ahd_outb(ahd, CLRINT, CLRSCSIINT);
 2856        ahd_unpause(ahd);
 2857}
 2858
 2859static void
 2860ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
 2861{
 2862        /*
 2863         * Clear the sources of the interrupts.
 2864         */
 2865        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 2866        ahd_outb(ahd, CLRLQIINT1, lqistat1);
 2867
 2868        /*
 2869         * If the "illegal" phase changes were in response
 2870         * to our ATN to flag a CRC error, AND we ended up
 2871         * on packet boundaries, clear the error, restart the
 2872         * LQI manager as appropriate, and go on our merry
 2873         * way toward sending the message.  Otherwise, reset
 2874         * the bus to clear the error.
 2875         */
 2876        ahd_set_active_fifo(ahd);
 2877        if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
 2878         && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
 2879                if ((lqistat1 & LQIPHASE_LQ) != 0) {
 2880                        printf("LQIRETRY for LQIPHASE_LQ\n");
 2881                        ahd_outb(ahd, LQCTL2, LQIRETRY);
 2882                } else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
 2883                        printf("LQIRETRY for LQIPHASE_NLQ\n");
 2884                        ahd_outb(ahd, LQCTL2, LQIRETRY);
 2885                } else
 2886                        panic("ahd_handle_lqiphase_error: No phase errors\n");
 2887                ahd_dump_card_state(ahd);
 2888                ahd_outb(ahd, CLRINT, CLRSCSIINT);
 2889                ahd_unpause(ahd);
 2890        } else {
 2891                printf("Reseting Channel for LQI Phase error\n");
 2892                ahd_dump_card_state(ahd);
 2893                ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
 2894        }
 2895}
 2896
 2897/*
 2898 * Packetized unexpected or expected busfree.
 2899 * Entered in mode based on busfreetime.
 2900 */
 2901static int
 2902ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
 2903{
 2904        u_int lqostat1;
 2905
 2906        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 2907                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 2908        lqostat1 = ahd_inb(ahd, LQOSTAT1);
 2909        if ((lqostat1 & LQOBUSFREE) != 0) {
 2910                struct scb *scb;
 2911                u_int scbid;
 2912                u_int saved_scbptr;
 2913                u_int waiting_h;
 2914                u_int waiting_t;
 2915                u_int next;
 2916
 2917                /*
 2918                 * The LQO manager detected an unexpected busfree
 2919                 * either:
 2920                 *
 2921                 * 1) During an outgoing LQ.
 2922                 * 2) After an outgoing LQ but before the first
 2923                 *    REQ of the command packet.
 2924                 * 3) During an outgoing command packet.
 2925                 *
 2926                 * In all cases, CURRSCB is pointing to the
 2927                 * SCB that encountered the failure.  Clean
 2928                 * up the queue, clear SELDO and LQOBUSFREE,
 2929                 * and allow the sequencer to restart the select
 2930                 * out at its lesure.
 2931                 */
 2932                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 2933                scbid = ahd_inw(ahd, CURRSCB);
 2934                scb = ahd_lookup_scb(ahd, scbid);
 2935                if (scb == NULL)
 2936                       panic("SCB not valid during LQOBUSFREE");
 2937                /*
 2938                 * Clear the status.
 2939                 */
 2940                ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
 2941                if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
 2942                        ahd_outb(ahd, CLRLQOINT1, 0);
 2943                ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
 2944                ahd_flush_device_writes(ahd);
 2945                ahd_outb(ahd, CLRSINT0, CLRSELDO);
 2946
 2947                /*
 2948                 * Return the LQO manager to its idle loop.  It will
 2949                 * not do this automatically if the busfree occurs
 2950                 * after the first REQ of either the LQ or command
 2951                 * packet or between the LQ and command packet.
 2952                 */
 2953                ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
 2954
 2955                /*
 2956                 * Update the waiting for selection queue so
 2957                 * we restart on the correct SCB.
 2958                 */
 2959                waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
 2960                saved_scbptr = ahd_get_scbptr(ahd);
 2961                if (waiting_h != scbid) {
 2962
 2963                        ahd_outw(ahd, WAITING_TID_HEAD, scbid);
 2964                        waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
 2965                        if (waiting_t == waiting_h) {
 2966                                ahd_outw(ahd, WAITING_TID_TAIL, scbid);
 2967                                next = SCB_LIST_NULL;
 2968                        } else {
 2969                                ahd_set_scbptr(ahd, waiting_h);
 2970                                next = ahd_inw_scbram(ahd, SCB_NEXT2);
 2971                        }
 2972                        ahd_set_scbptr(ahd, scbid);
 2973                        ahd_outw(ahd, SCB_NEXT2, next);
 2974                }
 2975                ahd_set_scbptr(ahd, saved_scbptr);
 2976                if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
 2977                        if (SCB_IS_SILENT(scb) == FALSE) {
 2978                                ahd_print_path(ahd, scb);
 2979                                printf("Probable outgoing LQ CRC error.  "
 2980                                       "Retrying command\n");
 2981                        }
 2982                        scb->crc_retry_count++;
 2983                } else {
 2984                        ahd_set_transaction_status(scb, CAM_UNCOR_PARITY);
 2985                        ahd_freeze_scb(scb);
 2986                        ahd_freeze_devq(ahd, scb);
 2987                }
 2988                /* Return unpausing the sequencer. */
 2989                return (0);
 2990        } else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
 2991                /*
 2992                 * Ignore what are really parity errors that
 2993                 * occur on the last REQ of a free running
 2994                 * clock prior to going busfree.  Some drives
 2995                 * do not properly active negate just before
 2996                 * going busfree resulting in a parity glitch.
 2997                 */
 2998                ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
 2999#ifdef AHD_DEBUG
 3000                if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
 3001                        printf("%s: Parity on last REQ detected "
 3002                               "during busfree phase.\n",
 3003                               ahd_name(ahd));
 3004#endif
 3005                /* Return unpausing the sequencer. */
 3006                return (0);
 3007        }
 3008        if (ahd->src_mode != AHD_MODE_SCSI) {
 3009                u_int        scbid;
 3010                struct        scb *scb;
 3011
 3012                scbid = ahd_get_scbptr(ahd);
 3013                scb = ahd_lookup_scb(ahd, scbid);
 3014                ahd_print_path(ahd, scb);
 3015                printf("Unexpected PKT busfree condition\n");
 3016                ahd_dump_card_state(ahd);
 3017                ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
 3018                               SCB_GET_LUN(scb), SCB_GET_TAG(scb),
 3019                               ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
 3020
 3021                /* Return restarting the sequencer. */
 3022                return (1);
 3023        }
 3024        printf("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
 3025        ahd_dump_card_state(ahd);
 3026        /* Restart the sequencer. */
 3027        return (1);
 3028}
 3029
 3030/*
 3031 * Non-packetized unexpected or expected busfree.
 3032 */
 3033static int
 3034ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
 3035{
 3036        struct        ahd_devinfo devinfo;
 3037        struct        scb *scb;
 3038        u_int        lastphase;
 3039        u_int        saved_scsiid;
 3040        u_int        saved_lun;
 3041        u_int        target;
 3042        u_int        initiator_role_id;
 3043        u_int        scbid;
 3044        u_int        ppr_busfree;
 3045        int        printerror;
 3046
 3047        /*
 3048         * Look at what phase we were last in.  If its message out,
 3049         * chances are pretty good that the busfree was in response
 3050         * to one of our abort requests.
 3051         */
 3052        lastphase = ahd_inb(ahd, LASTPHASE);
 3053        saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
 3054        saved_lun = ahd_inb(ahd, SAVED_LUN);
 3055        target = SCSIID_TARGET(ahd, saved_scsiid);
 3056        initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
 3057        ahd_compile_devinfo(&devinfo, initiator_role_id,
 3058                            target, saved_lun, 'A', ROLE_INITIATOR);
 3059        printerror = 1;
 3060
 3061        scbid = ahd_get_scbptr(ahd);
 3062        scb = ahd_lookup_scb(ahd, scbid);
 3063        if (scb != NULL
 3064         && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
 3065                scb = NULL;
 3066
 3067        ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
 3068        if (lastphase == P_MESGOUT) {
 3069                u_int tag;
 3070
 3071                tag = SCB_LIST_NULL;
 3072                if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
 3073                 || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
 3074                        int found;
 3075                        int sent_msg;
 3076
 3077                        if (scb == NULL) {
 3078                                ahd_print_devinfo(ahd, &devinfo);
 3079                                printf("Abort for unidentified "
 3080                                       "connection completed.\n");
 3081                                /* restart the sequencer. */
 3082                                return (1);
 3083                        }
 3084                        sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
 3085                        ahd_print_path(ahd, scb);
 3086                        printf("SCB %d - Abort%s Completed.\n",
 3087                               SCB_GET_TAG(scb),
 3088                               sent_msg == MSG_ABORT_TAG ? "" : " Tag");
 3089
 3090                        if (sent_msg == MSG_ABORT_TAG)
 3091                                tag = SCB_GET_TAG(scb);
 3092
 3093                        if ((scb->flags & SCB_EXTERNAL_RESET) != 0) {
 3094                                /*
 3095                                 * This abort is in response to an
 3096                                 * unexpected switch to command phase
 3097                                 * for a packetized connection.  Since
 3098                                 * the identify message was never sent,
 3099                                 * "saved lun" is 0.  We really want to
 3100                                 * abort only the SCB that encountered
 3101                                 * this error, which could have a different
 3102                                 * lun.  The SCB will be retried so the OS
 3103                                 * will see the UA after renegotiating to
 3104                                 * packetized.
 3105                                 */
 3106                                tag = SCB_GET_TAG(scb);
 3107                                saved_lun = scb->hscb->lun;
 3108                        }
 3109                        found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
 3110                                               tag, ROLE_INITIATOR,
 3111                                               CAM_REQ_ABORTED);
 3112                        printf("found == 0x%x\n", found);
 3113                        printerror = 0;
 3114                } else if (ahd_sent_msg(ahd, AHDMSG_1B,
 3115                                        MSG_BUS_DEV_RESET, TRUE)) {
 3116#ifdef __FreeBSD__
 3117                        /*
 3118                         * Don't mark the user's request for this BDR
 3119                         * as completing with CAM_BDR_SENT.  CAM3
 3120                         * specifies CAM_REQ_CMP.
 3121                         */
 3122                        if (scb != NULL
 3123                         && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
 3124                         && ahd_match_scb(ahd, scb, target, 'A',
 3125                                          CAM_LUN_WILDCARD, SCB_LIST_NULL,
 3126                                          ROLE_INITIATOR))
 3127                                ahd_set_transaction_status(scb, CAM_REQ_CMP);
 3128#endif
 3129                        ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
 3130                                            CAM_BDR_SENT, "Bus Device Reset",
 3131                                            /*verbose_level*/0);
 3132                        printerror = 0;
 3133                } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
 3134                        && ppr_busfree == 0) {
 3135                        struct ahd_initiator_tinfo *tinfo;
 3136                        struct ahd_tmode_tstate *tstate;
 3137
 3138                        /*
 3139                         * PPR Rejected.
 3140                         *
 3141                         * If the previous negotiation was packetized,
 3142                         * this could be because the device has been
 3143                         * reset without our knowledge.  Force our
 3144                         * current negotiation to async and retry the
 3145                         * negotiation.  Otherwise retry the command
 3146                         * with non-ppr negotiation.
 3147                         */
 3148#ifdef AHD_DEBUG
 3149                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 3150                                printf("PPR negotiation rejected busfree.\n");
 3151#endif
 3152                        tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
 3153                                                    devinfo.our_scsiid,
 3154                                                    devinfo.target, &tstate);
 3155                        if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
 3156                                ahd_set_width(ahd, &devinfo,
 3157                                              MSG_EXT_WDTR_BUS_8_BIT,
 3158                                              AHD_TRANS_CUR,
 3159                                              /*paused*/TRUE);
 3160                                ahd_set_syncrate(ahd, &devinfo,
 3161                                                /*period*/0, /*offset*/0,
 3162                                                /*ppr_options*/0,
 3163                                                AHD_TRANS_CUR,
 3164                                                /*paused*/TRUE);
 3165                                /*
 3166                                 * The expect PPR busfree handler below
 3167                                 * will effect the retry and necessary
 3168                                 * abort.
 3169                                 */
 3170                        } else {
 3171                                tinfo->curr.transport_version = 2;
 3172                                tinfo->goal.transport_version = 2;
 3173                                tinfo->goal.ppr_options = 0;
 3174                                /*
 3175                                 * Remove any SCBs in the waiting for selection
 3176                                 * queue that may also be for this target so
 3177                                 * that command ordering is preserved.
 3178                                 */
 3179                                ahd_freeze_devq(ahd, scb);
 3180                                ahd_qinfifo_requeue_tail(ahd, scb);
 3181                                printerror = 0;
 3182                        }
 3183                } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
 3184                        && ppr_busfree == 0) {
 3185                        /*
 3186                         * Negotiation Rejected.  Go-narrow and
 3187                         * retry command.
 3188                         */
 3189#ifdef AHD_DEBUG
 3190                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 3191                                printf("WDTR negotiation rejected busfree.\n");
 3192#endif
 3193                        ahd_set_width(ahd, &devinfo,
 3194                                      MSG_EXT_WDTR_BUS_8_BIT,
 3195                                      AHD_TRANS_CUR|AHD_TRANS_GOAL,
 3196                                      /*paused*/TRUE);
 3197                        /*
 3198                         * Remove any SCBs in the waiting for selection
 3199                         * queue that may also be for this target so that
 3200                         * command ordering is preserved.
 3201                         */
 3202                        ahd_freeze_devq(ahd, scb);
 3203                        ahd_qinfifo_requeue_tail(ahd, scb);
 3204                        printerror = 0;
 3205                } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
 3206                        && ppr_busfree == 0) {
 3207                        /*
 3208                         * Negotiation Rejected.  Go-async and
 3209                         * retry command.
 3210                         */
 3211#ifdef AHD_DEBUG
 3212                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 3213                                printf("SDTR negotiation rejected busfree.\n");
 3214#endif
 3215                        ahd_set_syncrate(ahd, &devinfo,
 3216                                        /*period*/0, /*offset*/0,
 3217                                        /*ppr_options*/0,
 3218                                        AHD_TRANS_CUR|AHD_TRANS_GOAL,
 3219                                        /*paused*/TRUE);
 3220                        /*
 3221                         * Remove any SCBs in the waiting for selection
 3222                         * queue that may also be for this target so that
 3223                         * command ordering is preserved.
 3224                         */
 3225                        ahd_freeze_devq(ahd, scb);
 3226                        ahd_qinfifo_requeue_tail(ahd, scb);
 3227                        printerror = 0;
 3228                } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
 3229                        && ahd_sent_msg(ahd, AHDMSG_1B,
 3230                                         MSG_INITIATOR_DET_ERR, TRUE)) {
 3231
 3232#ifdef AHD_DEBUG
 3233                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 3234                                printf("Expected IDE Busfree\n");
 3235#endif
 3236                        printerror = 0;
 3237                } else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
 3238                        && ahd_sent_msg(ahd, AHDMSG_1B,
 3239                                        MSG_MESSAGE_REJECT, TRUE)) {
 3240
 3241#ifdef AHD_DEBUG
 3242                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 3243                                printf("Expected QAS Reject Busfree\n");
 3244#endif
 3245                        printerror = 0;
 3246                }
 3247        }
 3248
 3249        /*
 3250         * The busfree required flag is honored at the end of
 3251         * the message phases.  We check it last in case we
 3252         * had to send some other message that caused a busfree.
 3253         */
 3254        if (printerror != 0
 3255         && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
 3256         && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
 3257
 3258                ahd_freeze_devq(ahd, scb);
 3259                ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
 3260                ahd_freeze_scb(scb);
 3261                if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
 3262                        ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
 3263                                       SCB_GET_CHANNEL(ahd, scb),
 3264                                       SCB_GET_LUN(scb), SCB_LIST_NULL,
 3265                                       ROLE_INITIATOR, CAM_REQ_ABORTED);
 3266                } else {
 3267#ifdef AHD_DEBUG
 3268                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 3269                                printf("PPR Negotiation Busfree.\n");
 3270#endif
 3271                        ahd_done(ahd, scb);
 3272                }
 3273                printerror = 0;
 3274        }
 3275        if (printerror != 0) {
 3276                int aborted;
 3277
 3278                aborted = 0;
 3279                if (scb != NULL) {
 3280                        u_int tag;
 3281
 3282                        if ((scb->hscb->control & TAG_ENB) != 0)
 3283                                tag = SCB_GET_TAG(scb);
 3284                        else
 3285                                tag = SCB_LIST_NULL;
 3286                        ahd_print_path(ahd, scb);
 3287                        aborted = ahd_abort_scbs(ahd, target, 'A',
 3288                                       SCB_GET_LUN(scb), tag,
 3289                                       ROLE_INITIATOR,
 3290                                       CAM_UNEXP_BUSFREE);
 3291                } else {
 3292                        /*
 3293                         * We had not fully identified this connection,
 3294                         * so we cannot abort anything.
 3295                         */
 3296                        printf("%s: ", ahd_name(ahd));
 3297                }
 3298                printf("Unexpected busfree %s, %d SCBs aborted, "
 3299                       "PRGMCNT == 0x%x\n",
 3300                       ahd_lookup_phase_entry(lastphase)->phasemsg,
 3301                       aborted,
 3302                       ahd_inw(ahd, PRGMCNT));
 3303                ahd_dump_card_state(ahd);
 3304                if (lastphase != P_BUSFREE)
 3305                        ahd_force_renegotiation(ahd, &devinfo);
 3306        }
 3307        /* Always restart the sequencer. */
 3308        return (1);
 3309}
 3310
 3311static void
 3312ahd_handle_proto_violation(struct ahd_softc *ahd)
 3313{
 3314        struct        ahd_devinfo devinfo;
 3315        struct        scb *scb;
 3316        u_int        scbid;
 3317        u_int        seq_flags;
 3318        u_int        curphase;
 3319        u_int        lastphase;
 3320        int        found;
 3321
 3322        ahd_fetch_devinfo(ahd, &devinfo);
 3323        scbid = ahd_get_scbptr(ahd);
 3324        scb = ahd_lookup_scb(ahd, scbid);
 3325        seq_flags = ahd_inb(ahd, SEQ_FLAGS);
 3326        curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
 3327        lastphase = ahd_inb(ahd, LASTPHASE);
 3328        if ((seq_flags & NOT_IDENTIFIED) != 0) {
 3329
 3330                /*
 3331                 * The reconnecting target either did not send an
 3332                 * identify message, or did, but we didn't find an SCB
 3333                 * to match.
 3334                 */
 3335                ahd_print_devinfo(ahd, &devinfo);
 3336                printf("Target did not send an IDENTIFY message. "
 3337                       "LASTPHASE = 0x%x.\n", lastphase);
 3338                scb = NULL;
 3339        } else if (scb == NULL) {
 3340                /*
 3341                 * We don't seem to have an SCB active for this
 3342                 * transaction.  Print an error and reset the bus.
 3343                 */
 3344                ahd_print_devinfo(ahd, &devinfo);
 3345                printf("No SCB found during protocol violation\n");
 3346                goto proto_violation_reset;
 3347        } else {
 3348                ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
 3349                if ((seq_flags & NO_CDB_SENT) != 0) {
 3350                        ahd_print_path(ahd, scb);
 3351                        printf("No or incomplete CDB sent to device.\n");
 3352                } else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
 3353                          & STATUS_RCVD) == 0) {
 3354                        /*
 3355                         * The target never bothered to provide status to
 3356                         * us prior to completing the command.  Since we don't
 3357                         * know the disposition of this command, we must attempt
 3358                         * to abort it.  Assert ATN and prepare to send an abort
 3359                         * message.
 3360                         */
 3361                        ahd_print_path(ahd, scb);
 3362                        printf("Completed command without status.\n");
 3363                } else {
 3364                        ahd_print_path(ahd, scb);
 3365                        printf("Unknown protocol violation.\n");
 3366                        ahd_dump_card_state(ahd);
 3367                }
 3368        }
 3369        if ((lastphase & ~P_DATAIN_DT) == 0
 3370         || lastphase == P_COMMAND) {
 3371proto_violation_reset:
 3372                /*
 3373                 * Target either went directly to data
 3374                 * phase or didn't respond to our ATN.
 3375                 * The only safe thing to do is to blow
 3376                 * it away with a bus reset.
 3377                 */
 3378                found = ahd_reset_channel(ahd, 'A', TRUE);
 3379                printf("%s: Issued Channel %c Bus Reset. "
 3380                       "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
 3381        } else {
 3382                /*
 3383                 * Leave the selection hardware off in case
 3384                 * this abort attempt will affect yet to
 3385                 * be sent commands.
 3386                 */
 3387                ahd_outb(ahd, SCSISEQ0,
 3388                         ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
 3389                ahd_assert_atn(ahd);
 3390                ahd_outb(ahd, MSG_OUT, HOST_MSG);
 3391                if (scb == NULL) {
 3392                        ahd_print_devinfo(ahd, &devinfo);
 3393                        ahd->msgout_buf[0] = MSG_ABORT_TASK;
 3394                        ahd->msgout_len = 1;
 3395                        ahd->msgout_index = 0;
 3396                        ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 3397                } else {
 3398                        ahd_print_path(ahd, scb);
 3399                        scb->flags |= SCB_ABORT;
 3400                }
 3401                printf("Protocol violation %s.  Attempting to abort.\n",
 3402                       ahd_lookup_phase_entry(curphase)->phasemsg);
 3403        }
 3404}
 3405
 3406/*
 3407 * Force renegotiation to occur the next time we initiate
 3408 * a command to the current device.
 3409 */
 3410static void
 3411ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 3412{
 3413        struct        ahd_initiator_tinfo *targ_info;
 3414        struct        ahd_tmode_tstate *tstate;
 3415
 3416#ifdef AHD_DEBUG
 3417        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 3418                ahd_print_devinfo(ahd, devinfo);
 3419                printf("Forcing renegotiation\n");
 3420        }
 3421#endif
 3422        targ_info = ahd_fetch_transinfo(ahd,
 3423                                        devinfo->channel,
 3424                                        devinfo->our_scsiid,
 3425                                        devinfo->target,
 3426                                        &tstate);
 3427        ahd_update_neg_request(ahd, devinfo, tstate,
 3428                               targ_info, AHD_NEG_IF_NON_ASYNC);
 3429}
 3430
 3431#define AHD_MAX_STEPS 2000
 3432static void
 3433ahd_clear_critical_section(struct ahd_softc *ahd)
 3434{
 3435        ahd_mode_state        saved_modes;
 3436        int                stepping;
 3437        int                steps;
 3438        int                first_instr;
 3439        u_int                simode0;
 3440        u_int                simode1;
 3441        u_int                simode3;
 3442        u_int                lqimode0;
 3443        u_int                lqimode1;
 3444        u_int                lqomode0;
 3445        u_int                lqomode1;
 3446
 3447        if (ahd->num_critical_sections == 0)
 3448                return;
 3449
 3450        stepping = FALSE;
 3451        steps = 0;
 3452        first_instr = 0;
 3453        simode0 = 0;
 3454        simode1 = 0;
 3455        simode3 = 0;
 3456        lqimode0 = 0;
 3457        lqimode1 = 0;
 3458        lqomode0 = 0;
 3459        lqomode1 = 0;
 3460        saved_modes = ahd_save_modes(ahd);
 3461        for (;;) {
 3462                struct        cs *cs;
 3463                u_int        seqaddr;
 3464                u_int        i;
 3465
 3466                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 3467                seqaddr = ahd_inw(ahd, CURADDR);
 3468
 3469                cs = ahd->critical_sections;
 3470                for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
 3471                        
 3472                        if (cs->begin < seqaddr && cs->end >= seqaddr)
 3473                                break;
 3474                }
 3475
 3476                if (i == ahd->num_critical_sections)
 3477                        break;
 3478
 3479                if (steps > AHD_MAX_STEPS) {
 3480                        printf("%s: Infinite loop in critical section\n"
 3481                               "%s: First Instruction 0x%x now 0x%x\n",
 3482                               ahd_name(ahd), ahd_name(ahd), first_instr,
 3483                               seqaddr);
 3484                        ahd_dump_card_state(ahd);
 3485                        panic("critical section loop");
 3486                }
 3487
 3488                steps++;
 3489#ifdef AHD_DEBUG
 3490                if ((ahd_debug & AHD_SHOW_MISC) != 0)
 3491                        printf("%s: Single stepping at 0x%x\n", ahd_name(ahd),
 3492                               seqaddr);
 3493#endif
 3494                if (stepping == FALSE) {
 3495
 3496                        first_instr = seqaddr;
 3497                          ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 3498                          simode0 = ahd_inb(ahd, SIMODE0);
 3499                        simode3 = ahd_inb(ahd, SIMODE3);
 3500                        lqimode0 = ahd_inb(ahd, LQIMODE0);
 3501                        lqimode1 = ahd_inb(ahd, LQIMODE1);
 3502                        lqomode0 = ahd_inb(ahd, LQOMODE0);
 3503                        lqomode1 = ahd_inb(ahd, LQOMODE1);
 3504                        ahd_outb(ahd, SIMODE0, 0);
 3505                        ahd_outb(ahd, SIMODE3, 0);
 3506                        ahd_outb(ahd, LQIMODE0, 0);
 3507                        ahd_outb(ahd, LQIMODE1, 0);
 3508                        ahd_outb(ahd, LQOMODE0, 0);
 3509                        ahd_outb(ahd, LQOMODE1, 0);
 3510                        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 3511                        simode1 = ahd_inb(ahd, SIMODE1);
 3512                        /*
 3513                         * We don't clear ENBUSFREE.  Unfortunately
 3514                         * we cannot re-enable busfree detection within
 3515                         * the current connection, so we must leave it
 3516                         * on while single stepping.
 3517                         */
 3518                        ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
 3519                        ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
 3520                        stepping = TRUE;
 3521                }
 3522                ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
 3523                ahd_outb(ahd, CLRINT, CLRSCSIINT);
 3524                ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
 3525                ahd_outb(ahd, HCNTRL, ahd->unpause);
 3526                while (!ahd_is_paused(ahd))
 3527                        ahd_delay(200);
 3528                ahd_update_modes(ahd);
 3529        }
 3530        if (stepping) {
 3531                ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 3532                ahd_outb(ahd, SIMODE0, simode0);
 3533                ahd_outb(ahd, SIMODE3, simode3);
 3534                ahd_outb(ahd, LQIMODE0, lqimode0);
 3535                ahd_outb(ahd, LQIMODE1, lqimode1);
 3536                ahd_outb(ahd, LQOMODE0, lqomode0);
 3537                ahd_outb(ahd, LQOMODE1, lqomode1);
 3538                ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 3539                ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
 3540                  ahd_outb(ahd, SIMODE1, simode1);
 3541                /*
 3542                 * SCSIINT seems to glitch occassionally when
 3543                 * the interrupt masks are restored.  Clear SCSIINT
 3544                 * one more time so that only persistent errors
 3545                 * are seen as a real interrupt.
 3546                 */
 3547                ahd_outb(ahd, CLRINT, CLRSCSIINT);
 3548        }
 3549        ahd_restore_modes(ahd, saved_modes);
 3550}
 3551
 3552/*
 3553 * Clear any pending interrupt status.
 3554 */
 3555static void
 3556ahd_clear_intstat(struct ahd_softc *ahd)
 3557{
 3558        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 3559                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 3560        /* Clear any interrupt conditions this may have caused */
 3561        ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
 3562                                 |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
 3563        ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
 3564                                 |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
 3565                                 |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
 3566        ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
 3567                                 |CLRLQOATNPKT|CLRLQOTCRC);
 3568        ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
 3569                                 |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
 3570        if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
 3571                ahd_outb(ahd, CLRLQOINT0, 0);
 3572                ahd_outb(ahd, CLRLQOINT1, 0);
 3573        }
 3574        ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
 3575        ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
 3576                                |CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
 3577        ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
 3578                                |CLRIOERR|CLROVERRUN);
 3579        ahd_outb(ahd, CLRINT, CLRSCSIINT);
 3580}
 3581
 3582/**************************** Debugging Routines ******************************/
 3583#ifdef AHD_DEBUG
 3584uint32_t ahd_debug = AHD_DEBUG_OPTS;
 3585#endif
 3586
 3587#if 0
 3588void
 3589ahd_print_scb(struct scb *scb)
 3590{
 3591        struct hardware_scb *hscb;
 3592        int i;
 3593
 3594        hscb = scb->hscb;
 3595        printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
 3596               (void *)scb,
 3597               hscb->control,
 3598               hscb->scsiid,
 3599               hscb->lun,
 3600               hscb->cdb_len);
 3601        printf("Shared Data: ");
 3602        for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
 3603                printf("%#02x", hscb->shared_data.idata.cdb[i]);
 3604        printf("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
 3605               (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
 3606               (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
 3607               ahd_le32toh(hscb->datacnt),
 3608               ahd_le32toh(hscb->sgptr),
 3609               SCB_GET_TAG(scb));
 3610        ahd_dump_sglist(scb);
 3611}
 3612#endif  /*  0  */
 3613
 3614/************************* Transfer Negotiation *******************************/
 3615/*
 3616 * Allocate per target mode instance (ID we respond to as a target)
 3617 * transfer negotiation data structures.
 3618 */
 3619static struct ahd_tmode_tstate *
 3620ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
 3621{
 3622        struct ahd_tmode_tstate *master_tstate;
 3623        struct ahd_tmode_tstate *tstate;
 3624        int i;
 3625
 3626        master_tstate = ahd->enabled_targets[ahd->our_id];
 3627        if (ahd->enabled_targets[scsi_id] != NULL
 3628         && ahd->enabled_targets[scsi_id] != master_tstate)
 3629                panic("%s: ahd_alloc_tstate - Target already allocated",
 3630                      ahd_name(ahd));
 3631        tstate = malloc(sizeof(*tstate), M_DEVBUF, M_NOWAIT);
 3632        if (tstate == NULL)
 3633                return (NULL);
 3634
 3635        /*
 3636         * If we have allocated a master tstate, copy user settings from
 3637         * the master tstate (taken from SRAM or the EEPROM) for this
 3638         * channel, but reset our current and goal settings to async/narrow
 3639         * until an initiator talks to us.
 3640         */
 3641        if (master_tstate != NULL) {
 3642                memcpy(tstate, master_tstate, sizeof(*tstate));
 3643                memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
 3644                for (i = 0; i < 16; i++) {
 3645                        memset(&tstate->transinfo[i].curr, 0,
 3646                              sizeof(tstate->transinfo[i].curr));
 3647                        memset(&tstate->transinfo[i].goal, 0,
 3648                              sizeof(tstate->transinfo[i].goal));
 3649                }
 3650        } else
 3651                memset(tstate, 0, sizeof(*tstate));
 3652        ahd->enabled_targets[scsi_id] = tstate;
 3653        return (tstate);
 3654}
 3655
 3656#ifdef AHD_TARGET_MODE
 3657/*
 3658 * Free per target mode instance (ID we respond to as a target)
 3659 * transfer negotiation data structures.
 3660 */
 3661static void
 3662ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
 3663{
 3664        struct ahd_tmode_tstate *tstate;
 3665
 3666        /*
 3667         * Don't clean up our "master" tstate.
 3668         * It has our default user settings.
 3669         */
 3670        if (scsi_id == ahd->our_id
 3671         && force == FALSE)
 3672                return;
 3673
 3674        tstate = ahd->enabled_targets[scsi_id];
 3675        if (tstate != NULL)
 3676                free(tstate, M_DEVBUF);
 3677        ahd->enabled_targets[scsi_id] = NULL;
 3678}
 3679#endif
 3680
 3681/*
 3682 * Called when we have an active connection to a target on the bus,
 3683 * this function finds the nearest period to the input period limited
 3684 * by the capabilities of the bus connectivity of and sync settings for
 3685 * the target.
 3686 */
 3687static void
 3688ahd_devlimited_syncrate(struct ahd_softc *ahd,
 3689                        struct ahd_initiator_tinfo *tinfo,
 3690                        u_int *period, u_int *ppr_options, role_t role)
 3691{
 3692        struct        ahd_transinfo *transinfo;
 3693        u_int        maxsync;
 3694
 3695        if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
 3696         && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
 3697                maxsync = AHD_SYNCRATE_PACED;
 3698        } else {
 3699                maxsync = AHD_SYNCRATE_ULTRA;
 3700                /* Can't do DT related options on an SE bus */
 3701                *ppr_options &= MSG_EXT_PPR_QAS_REQ;
 3702        }
 3703        /*
 3704         * Never allow a value higher than our current goal
 3705         * period otherwise we may allow a target initiated
 3706         * negotiation to go above the limit as set by the
 3707         * user.  In the case of an initiator initiated
 3708         * sync negotiation, we limit based on the user
 3709         * setting.  This allows the system to still accept
 3710         * incoming negotiations even if target initiated
 3711         * negotiation is not performed.
 3712         */
 3713        if (role == ROLE_TARGET)
 3714                transinfo = &tinfo->user;
 3715        else 
 3716                transinfo = &tinfo->goal;
 3717        *ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
 3718        if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
 3719                maxsync = max(maxsync, (u_int)AHD_SYNCRATE_ULTRA2);
 3720                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
 3721        }
 3722        if (transinfo->period == 0) {
 3723                *period = 0;
 3724                *ppr_options = 0;
 3725        } else {
 3726                *period = max(*period, (u_int)transinfo->period);
 3727                ahd_find_syncrate(ahd, period, ppr_options, maxsync);
 3728        }
 3729}
 3730
 3731/*
 3732 * Look up the valid period to SCSIRATE conversion in our table.
 3733 * Return the period and offset that should be sent to the target
 3734 * if this was the beginning of an SDTR.
 3735 */
 3736void
 3737ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
 3738                  u_int *ppr_options, u_int maxsync)
 3739{
 3740        if (*period < maxsync)
 3741                *period = maxsync;
 3742
 3743        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
 3744         && *period > AHD_SYNCRATE_MIN_DT)
 3745                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
 3746                
 3747        if (*period > AHD_SYNCRATE_MIN)
 3748                *period = 0;
 3749
 3750        /* Honor PPR option conformance rules. */
 3751        if (*period > AHD_SYNCRATE_PACED)
 3752                *ppr_options &= ~MSG_EXT_PPR_RTI;
 3753
 3754        if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
 3755                *ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
 3756
 3757        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
 3758                *ppr_options &= MSG_EXT_PPR_QAS_REQ;
 3759
 3760        /* Skip all PACED only entries if IU is not available */
 3761        if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
 3762         && *period < AHD_SYNCRATE_DT)
 3763                *period = AHD_SYNCRATE_DT;
 3764
 3765        /* Skip all DT only entries if DT is not available */
 3766        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
 3767         && *period < AHD_SYNCRATE_ULTRA2)
 3768                *period = AHD_SYNCRATE_ULTRA2;
 3769}
 3770
 3771/*
 3772 * Truncate the given synchronous offset to a value the
 3773 * current adapter type and syncrate are capable of.
 3774 */
 3775static void
 3776ahd_validate_offset(struct ahd_softc *ahd,
 3777                    struct ahd_initiator_tinfo *tinfo,
 3778                    u_int period, u_int *offset, int wide,
 3779                    role_t role)
 3780{
 3781        u_int maxoffset;
 3782
 3783        /* Limit offset to what we can do */
 3784        if (period == 0)
 3785                maxoffset = 0;
 3786        else if (period <= AHD_SYNCRATE_PACED) {
 3787                if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
 3788                        maxoffset = MAX_OFFSET_PACED_BUG;
 3789                else
 3790                        maxoffset = MAX_OFFSET_PACED;
 3791        } else
 3792                maxoffset = MAX_OFFSET_NON_PACED;
 3793        *offset = min(*offset, maxoffset);
 3794        if (tinfo != NULL) {
 3795                if (role == ROLE_TARGET)
 3796                        *offset = min(*offset, (u_int)tinfo->user.offset);
 3797                else
 3798                        *offset = min(*offset, (u_int)tinfo->goal.offset);
 3799        }
 3800}
 3801
 3802/*
 3803 * Truncate the given transfer width parameter to a value the
 3804 * current adapter type is capable of.
 3805 */
 3806static void
 3807ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
 3808                   u_int *bus_width, role_t role)
 3809{
 3810        switch (*bus_width) {
 3811        default:
 3812                if (ahd->features & AHD_WIDE) {
 3813                        /* Respond Wide */
 3814                        *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
 3815                        break;
 3816                }
 3817                /* FALLTHROUGH */
 3818        case MSG_EXT_WDTR_BUS_8_BIT:
 3819                *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
 3820                break;
 3821        }
 3822        if (tinfo != NULL) {
 3823                if (role == ROLE_TARGET)
 3824                        *bus_width = min((u_int)tinfo->user.width, *bus_width);
 3825                else
 3826                        *bus_width = min((u_int)tinfo->goal.width, *bus_width);
 3827        }
 3828}
 3829
 3830/*
 3831 * Update the bitmask of targets for which the controller should
 3832 * negotiate with at the next convenient oportunity.  This currently
 3833 * means the next time we send the initial identify messages for
 3834 * a new transaction.
 3835 */
 3836int
 3837ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 3838                       struct ahd_tmode_tstate *tstate,
 3839                       struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
 3840{
 3841        u_int auto_negotiate_orig;
 3842
 3843        auto_negotiate_orig = tstate->auto_negotiate;
 3844        if (neg_type == AHD_NEG_ALWAYS) {
 3845                /*
 3846                 * Force our "current" settings to be
 3847                 * unknown so that unless a bus reset
 3848                 * occurs the need to renegotiate is
 3849                 * recorded persistently.
 3850                 */
 3851                if ((ahd->features & AHD_WIDE) != 0)
 3852                        tinfo->curr.width = AHD_WIDTH_UNKNOWN;
 3853                tinfo->curr.period = AHD_PERIOD_UNKNOWN;
 3854                tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
 3855        }
 3856        if (tinfo->curr.period != tinfo->goal.period
 3857         || tinfo->curr.width != tinfo->goal.width
 3858         || tinfo->curr.offset != tinfo->goal.offset
 3859         || tinfo->curr.ppr_options != tinfo->goal.ppr_options
 3860         || (neg_type == AHD_NEG_IF_NON_ASYNC
 3861          && (tinfo->goal.offset != 0
 3862           || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
 3863           || tinfo->goal.ppr_options != 0)))
 3864                tstate->auto_negotiate |= devinfo->target_mask;
 3865        else
 3866                tstate->auto_negotiate &= ~devinfo->target_mask;
 3867
 3868        return (auto_negotiate_orig != tstate->auto_negotiate);
 3869}
 3870
 3871/*
 3872 * Update the user/goal/curr tables of synchronous negotiation
 3873 * parameters as well as, in the case of a current or active update,
 3874 * any data structures on the host controller.  In the case of an
 3875 * active update, the specified target is currently talking to us on
 3876 * the bus, so the transfer parameter update must take effect
 3877 * immediately.
 3878 */
 3879void
 3880ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 3881                 u_int period, u_int offset, u_int ppr_options,
 3882                 u_int type, int paused)
 3883{
 3884        struct        ahd_initiator_tinfo *tinfo;
 3885        struct        ahd_tmode_tstate *tstate;
 3886        u_int        old_period;
 3887        u_int        old_offset;
 3888        u_int        old_ppr;
 3889        int        active;
 3890        int        update_needed;
 3891
 3892        active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
 3893        update_needed = 0;
 3894
 3895        if (period == 0 || offset == 0) {
 3896                period = 0;
 3897                offset = 0;
 3898        }
 3899
 3900        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
 3901                                    devinfo->target, &tstate);
 3902
 3903        if ((type & AHD_TRANS_USER) != 0) {
 3904                tinfo->user.period = period;
 3905                tinfo->user.offset = offset;
 3906                tinfo->user.ppr_options = ppr_options;
 3907        }
 3908
 3909        if ((type & AHD_TRANS_GOAL) != 0) {
 3910                tinfo->goal.period = period;
 3911                tinfo->goal.offset = offset;
 3912                tinfo->goal.ppr_options = ppr_options;
 3913        }
 3914
 3915        old_period = tinfo->curr.period;
 3916        old_offset = tinfo->curr.offset;
 3917        old_ppr           = tinfo->curr.ppr_options;
 3918
 3919        if ((type & AHD_TRANS_CUR) != 0
 3920         && (old_period != period
 3921          || old_offset != offset
 3922          || old_ppr != ppr_options)) {
 3923
 3924                update_needed++;
 3925
 3926                tinfo->curr.period = period;
 3927                tinfo->curr.offset = offset;
 3928                tinfo->curr.ppr_options = ppr_options;
 3929
 3930                ahd_send_async(ahd, devinfo->channel, devinfo->target,
 3931                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
 3932                if (bootverbose) {
 3933                        if (offset != 0) {
 3934                                int options;
 3935
 3936                                printf("%s: target %d synchronous with "
 3937                                       "period = 0x%x, offset = 0x%x",
 3938                                       ahd_name(ahd), devinfo->target,
 3939                                       period, offset);
 3940                                options = 0;
 3941                                if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
 3942                                        printf("(RDSTRM");
 3943                                        options++;
 3944                                }
 3945                                if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
 3946                                        printf("%s", options ? "|DT" : "(DT");
 3947                                        options++;
 3948                                }
 3949                                if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
 3950                                        printf("%s", options ? "|IU" : "(IU");
 3951                                        options++;
 3952                                }
 3953                                if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
 3954                                        printf("%s", options ? "|RTI" : "(RTI");
 3955                                        options++;
 3956                                }
 3957                                if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
 3958                                        printf("%s", options ? "|QAS" : "(QAS");
 3959                                        options++;
 3960                                }
 3961                                if (options != 0)
 3962                                        printf(")\n");
 3963                                else
 3964                                        printf("\n");
 3965                        } else {
 3966                                printf("%s: target %d using "
 3967                                       "asynchronous transfers%s\n",
 3968                                       ahd_name(ahd), devinfo->target,
 3969                                       (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
 3970                                     ?  "(QAS)" : "");
 3971                        }
 3972                }
 3973        }
 3974        /*
 3975         * Always refresh the neg-table to handle the case of the
 3976         * sequencer setting the ENATNO bit for a MK_MESSAGE request.
 3977         * We will always renegotiate in that case if this is a
 3978         * packetized request.  Also manage the busfree expected flag
 3979         * from this common routine so that we catch changes due to
 3980         * WDTR or SDTR messages.
 3981         */
 3982        if ((type & AHD_TRANS_CUR) != 0) {
 3983                if (!paused)
 3984                        ahd_pause(ahd);
 3985                ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
 3986                if (!paused)
 3987                        ahd_unpause(ahd);
 3988                if (ahd->msg_type != MSG_TYPE_NONE) {
 3989                        if ((old_ppr & MSG_EXT_PPR_IU_REQ)
 3990                         != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
 3991#ifdef AHD_DEBUG
 3992                                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 3993                                        ahd_print_devinfo(ahd, devinfo);
 3994                                        printf("Expecting IU Change busfree\n");
 3995                                }
 3996#endif
 3997                                ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
 3998                                               |  MSG_FLAG_IU_REQ_CHANGED;
 3999                        }
 4000                        if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
 4001#ifdef AHD_DEBUG
 4002                                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 4003                                        printf("PPR with IU_REQ outstanding\n");
 4004#endif
 4005                                ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
 4006                        }
 4007                }
 4008        }
 4009
 4010        update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
 4011                                                tinfo, AHD_NEG_TO_GOAL);
 4012
 4013        if (update_needed && active)
 4014                ahd_update_pending_scbs(ahd);
 4015}
 4016
 4017/*
 4018 * Update the user/goal/curr tables of wide negotiation
 4019 * parameters as well as, in the case of a current or active update,
 4020 * any data structures on the host controller.  In the case of an
 4021 * active update, the specified target is currently talking to us on
 4022 * the bus, so the transfer parameter update must take effect
 4023 * immediately.
 4024 */
 4025void
 4026ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4027              u_int width, u_int type, int paused)
 4028{
 4029        struct        ahd_initiator_tinfo *tinfo;
 4030        struct        ahd_tmode_tstate *tstate;
 4031        u_int        oldwidth;
 4032        int        active;
 4033        int        update_needed;
 4034
 4035        active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
 4036        update_needed = 0;
 4037        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
 4038                                    devinfo->target, &tstate);
 4039
 4040        if ((type & AHD_TRANS_USER) != 0)
 4041                tinfo->user.width = width;
 4042
 4043        if ((type & AHD_TRANS_GOAL) != 0)
 4044                tinfo->goal.width = width;
 4045
 4046        oldwidth = tinfo->curr.width;
 4047        if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
 4048
 4049                update_needed++;
 4050
 4051                tinfo->curr.width = width;
 4052                ahd_send_async(ahd, devinfo->channel, devinfo->target,
 4053                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
 4054                if (bootverbose) {
 4055                        printf("%s: target %d using %dbit transfers\n",
 4056                               ahd_name(ahd), devinfo->target,
 4057                               8 * (0x01 << width));
 4058                }
 4059        }
 4060
 4061        if ((type & AHD_TRANS_CUR) != 0) {
 4062                if (!paused)
 4063                        ahd_pause(ahd);
 4064                ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
 4065                if (!paused)
 4066                        ahd_unpause(ahd);
 4067        }
 4068
 4069        update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
 4070                                                tinfo, AHD_NEG_TO_GOAL);
 4071        if (update_needed && active)
 4072                ahd_update_pending_scbs(ahd);
 4073
 4074}
 4075
 4076/*
 4077 * Update the current state of tagged queuing for a given target.
 4078 */
 4079static void
 4080ahd_set_tags(struct ahd_softc *ahd, struct scsi_cmnd *cmd,
 4081             struct ahd_devinfo *devinfo, ahd_queue_alg alg)
 4082{
 4083        struct scsi_device *sdev = cmd->device;
 4084
 4085        ahd_platform_set_tags(ahd, sdev, devinfo, alg);
 4086        ahd_send_async(ahd, devinfo->channel, devinfo->target,
 4087                       devinfo->lun, AC_TRANSFER_NEG);
 4088}
 4089
 4090static void
 4091ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4092                     struct ahd_transinfo *tinfo)
 4093{
 4094        ahd_mode_state        saved_modes;
 4095        u_int                period;
 4096        u_int                ppr_opts;
 4097        u_int                con_opts;
 4098        u_int                offset;
 4099        u_int                saved_negoaddr;
 4100        uint8_t                iocell_opts[sizeof(ahd->iocell_opts)];
 4101
 4102        saved_modes = ahd_save_modes(ahd);
 4103        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 4104
 4105        saved_negoaddr = ahd_inb(ahd, NEGOADDR);
 4106        ahd_outb(ahd, NEGOADDR, devinfo->target);
 4107        period = tinfo->period;
 4108        offset = tinfo->offset;
 4109        memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts)); 
 4110        ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
 4111                                        |MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
 4112        con_opts = 0;
 4113        if (period == 0)
 4114                period = AHD_SYNCRATE_ASYNC;
 4115        if (period == AHD_SYNCRATE_160) {
 4116
 4117                if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
 4118                        /*
 4119                         * When the SPI4 spec was finalized, PACE transfers
 4120                         * was not made a configurable option in the PPR
 4121                         * message.  Instead it is assumed to be enabled for
 4122                         * any syncrate faster than 80MHz.  Nevertheless,
 4123                         * Harpoon2A4 allows this to be configurable.
 4124                         *
 4125                         * Harpoon2A4 also assumes at most 2 data bytes per
 4126                         * negotiated REQ/ACK offset.  Paced transfers take
 4127                         * 4, so we must adjust our offset.
 4128                         */
 4129                        ppr_opts |= PPROPT_PACE;
 4130                        offset *= 2;
 4131
 4132                        /*
 4133                         * Harpoon2A assumed that there would be a
 4134                         * fallback rate between 160MHz and 80MHz,
 4135                         * so 7 is used as the period factor rather
 4136                         * than 8 for 160MHz.
 4137                         */
 4138                        period = AHD_SYNCRATE_REVA_160;
 4139                }
 4140                if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
 4141                        iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
 4142                            ~AHD_PRECOMP_MASK;
 4143        } else {
 4144                /*
 4145                 * Precomp should be disabled for non-paced transfers.
 4146                 */
 4147                iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
 4148
 4149                if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
 4150                 && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
 4151                 && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
 4152                        /*
 4153                         * Slow down our CRC interval to be
 4154                         * compatible with non-packetized
 4155                         * U160 devices that can't handle a
 4156                         * CRC at full speed.
 4157                         */
 4158                        con_opts |= ENSLOWCRC;
 4159                }
 4160
 4161                if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
 4162                        /*
 4163                         * On H2A4, revert to a slower slewrate
 4164                         * on non-paced transfers.
 4165                         */
 4166                        iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
 4167                            ~AHD_SLEWRATE_MASK;
 4168                }
 4169        }
 4170
 4171        ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
 4172        ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
 4173        ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
 4174        ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
 4175
 4176        ahd_outb(ahd, NEGPERIOD, period);
 4177        ahd_outb(ahd, NEGPPROPTS, ppr_opts);
 4178        ahd_outb(ahd, NEGOFFSET, offset);
 4179
 4180        if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
 4181                con_opts |= WIDEXFER;
 4182
 4183        /*
 4184         * Slow down our CRC interval to be
 4185         * compatible with packetized U320 devices
 4186         * that can't handle a CRC at full speed
 4187         */
 4188        if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
 4189                con_opts |= ENSLOWCRC;
 4190        }
 4191
 4192        /*
 4193         * During packetized transfers, the target will
 4194         * give us the oportunity to send command packets
 4195         * without us asserting attention.
 4196         */
 4197        if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
 4198                con_opts |= ENAUTOATNO;
 4199        ahd_outb(ahd, NEGCONOPTS, con_opts);
 4200        ahd_outb(ahd, NEGOADDR, saved_negoaddr);
 4201        ahd_restore_modes(ahd, saved_modes);
 4202}
 4203
 4204/*
 4205 * When the transfer settings for a connection change, setup for
 4206 * negotiation in pending SCBs to effect the change as quickly as
 4207 * possible.  We also cancel any negotiations that are scheduled
 4208 * for inflight SCBs that have not been started yet.
 4209 */
 4210static void
 4211ahd_update_pending_scbs(struct ahd_softc *ahd)
 4212{
 4213        struct                scb *pending_scb;
 4214        int                pending_scb_count;
 4215        int                paused;
 4216        u_int                saved_scbptr;
 4217        ahd_mode_state        saved_modes;
 4218
 4219        /*
 4220         * Traverse the pending SCB list and ensure that all of the
 4221         * SCBs there have the proper settings.  We can only safely
 4222         * clear the negotiation required flag (setting requires the
 4223         * execution queue to be modified) and this is only possible
 4224         * if we are not already attempting to select out for this
 4225         * SCB.  For this reason, all callers only call this routine
 4226         * if we are changing the negotiation settings for the currently
 4227         * active transaction on the bus.
 4228         */
 4229        pending_scb_count = 0;
 4230        LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
 4231                struct ahd_devinfo devinfo;
 4232                struct ahd_initiator_tinfo *tinfo;
 4233                struct ahd_tmode_tstate *tstate;
 4234
 4235                ahd_scb_devinfo(ahd, &devinfo, pending_scb);
 4236                tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
 4237                                            devinfo.our_scsiid,
 4238                                            devinfo.target, &tstate);
 4239                if ((tstate->auto_negotiate & devinfo.target_mask) == 0
 4240                 && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
 4241                        pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
 4242                        pending_scb->hscb->control &= ~MK_MESSAGE;
 4243                }
 4244                ahd_sync_scb(ahd, pending_scb,
 4245                             BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 4246                pending_scb_count++;
 4247        }
 4248
 4249        if (pending_scb_count == 0)
 4250                return;
 4251
 4252        if (ahd_is_paused(ahd)) {
 4253                paused = 1;
 4254        } else {
 4255                paused = 0;
 4256                ahd_pause(ahd);
 4257        }
 4258
 4259        /*
 4260         * Force the sequencer to reinitialize the selection for
 4261         * the command at the head of the execution queue if it
 4262         * has already been setup.  The negotiation changes may
 4263         * effect whether we select-out with ATN.  It is only
 4264         * safe to clear ENSELO when the bus is not free and no
 4265         * selection is in progres or completed.
 4266         */
 4267        saved_modes = ahd_save_modes(ahd);
 4268        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 4269        if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
 4270         && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
 4271                ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
 4272        saved_scbptr = ahd_get_scbptr(ahd);
 4273        /* Ensure that the hscbs down on the card match the new information */
 4274        LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
 4275                u_int        scb_tag;
 4276                u_int        control;
 4277
 4278                scb_tag = SCB_GET_TAG(pending_scb);
 4279                ahd_set_scbptr(ahd, scb_tag);
 4280                control = ahd_inb_scbram(ahd, SCB_CONTROL);
 4281                control &= ~MK_MESSAGE;
 4282                control |= pending_scb->hscb->control & MK_MESSAGE;
 4283                ahd_outb(ahd, SCB_CONTROL, control);
 4284        }
 4285        ahd_set_scbptr(ahd, saved_scbptr);
 4286        ahd_restore_modes(ahd, saved_modes);
 4287
 4288        if (paused == 0)
 4289                ahd_unpause(ahd);
 4290}
 4291
 4292/**************************** Pathing Information *****************************/
 4293static void
 4294ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 4295{
 4296        ahd_mode_state        saved_modes;
 4297        u_int                saved_scsiid;
 4298        role_t                role;
 4299        int                our_id;
 4300
 4301        saved_modes = ahd_save_modes(ahd);
 4302        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 4303
 4304        if (ahd_inb(ahd, SSTAT0) & TARGET)
 4305                role = ROLE_TARGET;
 4306        else
 4307                role = ROLE_INITIATOR;
 4308
 4309        if (role == ROLE_TARGET
 4310         && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
 4311                /* We were selected, so pull our id from TARGIDIN */
 4312                our_id = ahd_inb(ahd, TARGIDIN) & OID;
 4313        } else if (role == ROLE_TARGET)
 4314                our_id = ahd_inb(ahd, TOWNID);
 4315        else
 4316                our_id = ahd_inb(ahd, IOWNID);
 4317
 4318        saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
 4319        ahd_compile_devinfo(devinfo,
 4320                            our_id,
 4321                            SCSIID_TARGET(ahd, saved_scsiid),
 4322                            ahd_inb(ahd, SAVED_LUN),
 4323                            SCSIID_CHANNEL(ahd, saved_scsiid),
 4324                            role);
 4325        ahd_restore_modes(ahd, saved_modes);
 4326}
 4327
 4328void
 4329ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 4330{
 4331        printf("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
 4332               devinfo->target, devinfo->lun);
 4333}
 4334
 4335static const struct ahd_phase_table_entry*
 4336ahd_lookup_phase_entry(int phase)
 4337{
 4338        const struct ahd_phase_table_entry *entry;
 4339        const struct ahd_phase_table_entry *last_entry;
 4340
 4341        /*
 4342         * num_phases doesn't include the default entry which
 4343         * will be returned if the phase doesn't match.
 4344         */
 4345        last_entry = &ahd_phase_table[num_phases];
 4346        for (entry = ahd_phase_table; entry < last_entry; entry++) {
 4347                if (phase == entry->phase)
 4348                        break;
 4349        }
 4350        return (entry);
 4351}
 4352
 4353void
 4354ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
 4355                    u_int lun, char channel, role_t role)
 4356{
 4357        devinfo->our_scsiid = our_id;
 4358        devinfo->target = target;
 4359        devinfo->lun = lun;
 4360        devinfo->target_offset = target;
 4361        devinfo->channel = channel;
 4362        devinfo->role = role;
 4363        if (channel == 'B')
 4364                devinfo->target_offset += 8;
 4365        devinfo->target_mask = (0x01 << devinfo->target_offset);
 4366}
 4367
 4368static void
 4369ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4370                struct scb *scb)
 4371{
 4372        role_t        role;
 4373        int        our_id;
 4374
 4375        our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
 4376        role = ROLE_INITIATOR;
 4377        if ((scb->hscb->control & TARGET_SCB) != 0)
 4378                role = ROLE_TARGET;
 4379        ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
 4380                            SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
 4381}
 4382
 4383
 4384/************************ Message Phase Processing ****************************/
 4385/*
 4386 * When an initiator transaction with the MK_MESSAGE flag either reconnects
 4387 * or enters the initial message out phase, we are interrupted.  Fill our
 4388 * outgoing message buffer with the appropriate message and beging handing
 4389 * the message phase(s) manually.
 4390 */
 4391static void
 4392ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4393                           struct scb *scb)
 4394{
 4395        /*
 4396         * To facilitate adding multiple messages together,
 4397         * each routine should increment the index and len
 4398         * variables instead of setting them explicitly.
 4399         */
 4400        ahd->msgout_index = 0;
 4401        ahd->msgout_len = 0;
 4402
 4403        if (ahd_currently_packetized(ahd))
 4404                ahd->msg_flags |= MSG_FLAG_PACKETIZED;
 4405
 4406        if (ahd->send_msg_perror
 4407         && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
 4408                ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
 4409                ahd->msgout_len++;
 4410                ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 4411#ifdef AHD_DEBUG
 4412                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 4413                        printf("Setting up for Parity Error delivery\n");
 4414#endif
 4415                return;
 4416        } else if (scb == NULL) {
 4417                printf("%s: WARNING. No pending message for "
 4418                       "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
 4419                ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
 4420                ahd->msgout_len++;
 4421                ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 4422                return;
 4423        }
 4424
 4425        if ((scb->flags & SCB_DEVICE_RESET) == 0
 4426         && (scb->flags & SCB_PACKETIZED) == 0
 4427         && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
 4428                u_int identify_msg;
 4429
 4430                identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
 4431                if ((scb->hscb->control & DISCENB) != 0)
 4432                        identify_msg |= MSG_IDENTIFY_DISCFLAG;
 4433                ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
 4434                ahd->msgout_len++;
 4435
 4436                if ((scb->hscb->control & TAG_ENB) != 0) {
 4437                        ahd->msgout_buf[ahd->msgout_index++] =
 4438                            scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
 4439                        ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
 4440                        ahd->msgout_len += 2;
 4441                }
 4442        }
 4443
 4444        if (scb->flags & SCB_DEVICE_RESET) {
 4445                ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
 4446                ahd->msgout_len++;
 4447                ahd_print_path(ahd, scb);
 4448                printf("Bus Device Reset Message Sent\n");
 4449                /*
 4450                 * Clear our selection hardware in advance of
 4451                 * the busfree.  We may have an entry in the waiting
 4452                 * Q for this target, and we don't want to go about
 4453                 * selecting while we handle the busfree and blow it
 4454                 * away.
 4455                 */
 4456                ahd_outb(ahd, SCSISEQ0, 0);
 4457        } else if ((scb->flags & SCB_ABORT) != 0) {
 4458
 4459                if ((scb->hscb->control & TAG_ENB) != 0) {
 4460                        ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
 4461                } else {
 4462                        ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
 4463                }
 4464                ahd->msgout_len++;
 4465                ahd_print_path(ahd, scb);
 4466                printf("Abort%s Message Sent\n",
 4467                       (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
 4468                /*
 4469                 * Clear our selection hardware in advance of
 4470                 * the busfree.  We may have an entry in the waiting
 4471                 * Q for this target, and we don't want to go about
 4472                 * selecting while we handle the busfree and blow it
 4473                 * away.
 4474                 */
 4475                ahd_outb(ahd, SCSISEQ0, 0);
 4476        } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
 4477                ahd_build_transfer_msg(ahd, devinfo);
 4478                /*
 4479                 * Clear our selection hardware in advance of potential
 4480                 * PPR IU status change busfree.  We may have an entry in
 4481                 * the waiting Q for this target, and we don't want to go
 4482                 * about selecting while we handle the busfree and blow
 4483                 * it away.
 4484                 */
 4485                ahd_outb(ahd, SCSISEQ0, 0);
 4486        } else {
 4487                printf("ahd_intr: AWAITING_MSG for an SCB that "
 4488                       "does not have a waiting message\n");
 4489                printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
 4490                       devinfo->target_mask);
 4491                panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
 4492                      "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
 4493                      ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
 4494                      scb->flags);
 4495        }
 4496
 4497        /*
 4498         * Clear the MK_MESSAGE flag from the SCB so we aren't
 4499         * asked to send this message again.
 4500         */
 4501        ahd_outb(ahd, SCB_CONTROL,
 4502                 ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
 4503        scb->hscb->control &= ~MK_MESSAGE;
 4504        ahd->msgout_index = 0;
 4505        ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 4506}
 4507
 4508/*
 4509 * Build an appropriate transfer negotiation message for the
 4510 * currently active target.
 4511 */
 4512static void
 4513ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 4514{
 4515        /*
 4516         * We need to initiate transfer negotiations.
 4517         * If our current and goal settings are identical,
 4518         * we want to renegotiate due to a check condition.
 4519         */
 4520        struct        ahd_initiator_tinfo *tinfo;
 4521        struct        ahd_tmode_tstate *tstate;
 4522        int        dowide;
 4523        int        dosync;
 4524        int        doppr;
 4525        u_int        period;
 4526        u_int        ppr_options;
 4527        u_int        offset;
 4528
 4529        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
 4530                                    devinfo->target, &tstate);
 4531        /*
 4532         * Filter our period based on the current connection.
 4533         * If we can't perform DT transfers on this segment (not in LVD
 4534         * mode for instance), then our decision to issue a PPR message
 4535         * may change.
 4536         */
 4537        period = tinfo->goal.period;
 4538        offset = tinfo->goal.offset;
 4539        ppr_options = tinfo->goal.ppr_options;
 4540        /* Target initiated PPR is not allowed in the SCSI spec */
 4541        if (devinfo->role == ROLE_TARGET)
 4542                ppr_options = 0;
 4543        ahd_devlimited_syncrate(ahd, tinfo, &period,
 4544                                &ppr_options, devinfo->role);
 4545        dowide = tinfo->curr.width != tinfo->goal.width;
 4546        dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
 4547        /*
 4548         * Only use PPR if we have options that need it, even if the device
 4549         * claims to support it.  There might be an expander in the way
 4550         * that doesn't.
 4551         */
 4552        doppr = ppr_options != 0;
 4553
 4554        if (!dowide && !dosync && !doppr) {
 4555                dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
 4556                dosync = tinfo->goal.offset != 0;
 4557        }
 4558
 4559        if (!dowide && !dosync && !doppr) {
 4560                /*
 4561                 * Force async with a WDTR message if we have a wide bus,
 4562                 * or just issue an SDTR with a 0 offset.
 4563                 */
 4564                if ((ahd->features & AHD_WIDE) != 0)
 4565                        dowide = 1;
 4566                else
 4567                        dosync = 1;
 4568
 4569                if (bootverbose) {
 4570                        ahd_print_devinfo(ahd, devinfo);
 4571                        printf("Ensuring async\n");
 4572                }
 4573        }
 4574        /* Target initiated PPR is not allowed in the SCSI spec */
 4575        if (devinfo->role == ROLE_TARGET)
 4576                doppr = 0;
 4577
 4578        /*
 4579         * Both the PPR message and SDTR message require the
 4580         * goal syncrate to be limited to what the target device
 4581         * is capable of handling (based on whether an LVD->SE
 4582         * expander is on the bus), so combine these two cases.
 4583         * Regardless, guarantee that if we are using WDTR and SDTR
 4584         * messages that WDTR comes first.
 4585         */
 4586        if (doppr || (dosync && !dowide)) {
 4587
 4588                offset = tinfo->goal.offset;
 4589                ahd_validate_offset(ahd, tinfo, period, &offset,
 4590                                    doppr ? tinfo->goal.width
 4591                                          : tinfo->curr.width,
 4592                                    devinfo->role);
 4593                if (doppr) {
 4594                        ahd_construct_ppr(ahd, devinfo, period, offset,
 4595                                          tinfo->goal.width, ppr_options);
 4596                } else {
 4597                        ahd_construct_sdtr(ahd, devinfo, period, offset);
 4598                }
 4599        } else {
 4600                ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
 4601        }
 4602}
 4603
 4604/*
 4605 * Build a synchronous negotiation message in our message
 4606 * buffer based on the input parameters.
 4607 */
 4608static void
 4609ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4610                   u_int period, u_int offset)
 4611{
 4612        if (offset == 0)
 4613                period = AHD_ASYNC_XFER_PERIOD;
 4614        ahd->msgout_index += spi_populate_sync_msg(
 4615                        ahd->msgout_buf + ahd->msgout_index, period, offset);
 4616        ahd->msgout_len += 5;
 4617        if (bootverbose) {
 4618                printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
 4619                       ahd_name(ahd), devinfo->channel, devinfo->target,
 4620                       devinfo->lun, period, offset);
 4621        }
 4622}
 4623
 4624/*
 4625 * Build a wide negotiateion message in our message
 4626 * buffer based on the input parameters.
 4627 */
 4628static void
 4629ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4630                   u_int bus_width)
 4631{
 4632        ahd->msgout_index += spi_populate_width_msg(
 4633                        ahd->msgout_buf + ahd->msgout_index, bus_width);
 4634        ahd->msgout_len += 4;
 4635        if (bootverbose) {
 4636                printf("(%s:%c:%d:%d): Sending WDTR %x\n",
 4637                       ahd_name(ahd), devinfo->channel, devinfo->target,
 4638                       devinfo->lun, bus_width);
 4639        }
 4640}
 4641
 4642/*
 4643 * Build a parallel protocol request message in our message
 4644 * buffer based on the input parameters.
 4645 */
 4646static void
 4647ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 4648                  u_int period, u_int offset, u_int bus_width,
 4649                  u_int ppr_options)
 4650{
 4651        /*
 4652         * Always request precompensation from
 4653         * the other target if we are running
 4654         * at paced syncrates.
 4655         */
 4656        if (period <= AHD_SYNCRATE_PACED)
 4657                ppr_options |= MSG_EXT_PPR_PCOMP_EN;
 4658        if (offset == 0)
 4659                period = AHD_ASYNC_XFER_PERIOD;
 4660        ahd->msgout_index += spi_populate_ppr_msg(
 4661                        ahd->msgout_buf + ahd->msgout_index, period, offset,
 4662                        bus_width, ppr_options);
 4663        ahd->msgout_len += 8;
 4664        if (bootverbose) {
 4665                printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
 4666                       "offset %x, ppr_options %x\n", ahd_name(ahd),
 4667                       devinfo->channel, devinfo->target, devinfo->lun,
 4668                       bus_width, period, offset, ppr_options);
 4669        }
 4670}
 4671
 4672/*
 4673 * Clear any active message state.
 4674 */
 4675static void
 4676ahd_clear_msg_state(struct ahd_softc *ahd)
 4677{
 4678        ahd_mode_state saved_modes;
 4679
 4680        saved_modes = ahd_save_modes(ahd);
 4681        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 4682        ahd->send_msg_perror = 0;
 4683        ahd->msg_flags = MSG_FLAG_NONE;
 4684        ahd->msgout_len = 0;
 4685        ahd->msgin_index = 0;
 4686        ahd->msg_type = MSG_TYPE_NONE;
 4687        if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
 4688                /*
 4689                 * The target didn't care to respond to our
 4690                 * message request, so clear ATN.
 4691                 */
 4692                ahd_outb(ahd, CLRSINT1, CLRATNO);
 4693        }
 4694        ahd_outb(ahd, MSG_OUT, MSG_NOOP);
 4695        ahd_outb(ahd, SEQ_FLAGS2,
 4696                 ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
 4697        ahd_restore_modes(ahd, saved_modes);
 4698}
 4699
 4700/*
 4701 * Manual message loop handler.
 4702 */
 4703static void
 4704ahd_handle_message_phase(struct ahd_softc *ahd)
 4705{
 4706        struct        ahd_devinfo devinfo;
 4707        u_int        bus_phase;
 4708        int        end_session;
 4709
 4710        ahd_fetch_devinfo(ahd, &devinfo);
 4711        end_session = FALSE;
 4712        bus_phase = ahd_inb(ahd, LASTPHASE);
 4713
 4714        if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
 4715                printf("LQIRETRY for LQIPHASE_OUTPKT\n");
 4716                ahd_outb(ahd, LQCTL2, LQIRETRY);
 4717        }
 4718reswitch:
 4719        switch (ahd->msg_type) {
 4720        case MSG_TYPE_INITIATOR_MSGOUT:
 4721        {
 4722                int lastbyte;
 4723                int phasemis;
 4724                int msgdone;
 4725
 4726                if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
 4727                        panic("HOST_MSG_LOOP interrupt with no active message");
 4728
 4729#ifdef AHD_DEBUG
 4730                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 4731                        ahd_print_devinfo(ahd, &devinfo);
 4732                        printf("INITIATOR_MSG_OUT");
 4733                }
 4734#endif
 4735                phasemis = bus_phase != P_MESGOUT;
 4736                if (phasemis) {
 4737#ifdef AHD_DEBUG
 4738                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 4739                                printf(" PHASEMIS %s\n",
 4740                                       ahd_lookup_phase_entry(bus_phase)
 4741                                                             ->phasemsg);
 4742                        }
 4743#endif
 4744                        if (bus_phase == P_MESGIN) {
 4745                                /*
 4746                                 * Change gears and see if
 4747                                 * this messages is of interest to
 4748                                 * us or should be passed back to
 4749                                 * the sequencer.
 4750                                 */
 4751                                ahd_outb(ahd, CLRSINT1, CLRATNO);
 4752                                ahd->send_msg_perror = 0;
 4753                                ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
 4754                                ahd->msgin_index = 0;
 4755                                goto reswitch;
 4756                        }
 4757                        end_session = TRUE;
 4758                        break;
 4759                }
 4760
 4761                if (ahd->send_msg_perror) {
 4762                        ahd_outb(ahd, CLRSINT1, CLRATNO);
 4763                        ahd_outb(ahd, CLRSINT1, CLRREQINIT);
 4764#ifdef AHD_DEBUG
 4765                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 4766                                printf(" byte 0x%x\n", ahd->send_msg_perror);
 4767#endif
 4768                        /*
 4769                         * If we are notifying the target of a CRC error
 4770                         * during packetized operations, the target is
 4771                         * within its rights to acknowledge our message
 4772                         * with a busfree.
 4773                         */
 4774                        if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
 4775                         && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
 4776                                ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
 4777
 4778                        ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
 4779                        ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
 4780                        break;
 4781                }
 4782
 4783                msgdone        = ahd->msgout_index == ahd->msgout_len;
 4784                if (msgdone) {
 4785                        /*
 4786                         * The target has requested a retry.
 4787                         * Re-assert ATN, reset our message index to
 4788                         * 0, and try again.
 4789                         */
 4790                        ahd->msgout_index = 0;
 4791                        ahd_assert_atn(ahd);
 4792                }
 4793
 4794                lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
 4795                if (lastbyte) {
 4796                        /* Last byte is signified by dropping ATN */
 4797                        ahd_outb(ahd, CLRSINT1, CLRATNO);
 4798                }
 4799
 4800                /*
 4801                 * Clear our interrupt status and present
 4802                 * the next byte on the bus.
 4803                 */
 4804                ahd_outb(ahd, CLRSINT1, CLRREQINIT);
 4805#ifdef AHD_DEBUG
 4806                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 4807                        printf(" byte 0x%x\n",
 4808                               ahd->msgout_buf[ahd->msgout_index]);
 4809#endif
 4810                ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
 4811                ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
 4812                break;
 4813        }
 4814        case MSG_TYPE_INITIATOR_MSGIN:
 4815        {
 4816                int phasemis;
 4817                int message_done;
 4818
 4819#ifdef AHD_DEBUG
 4820                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 4821                        ahd_print_devinfo(ahd, &devinfo);
 4822                        printf("INITIATOR_MSG_IN");
 4823                }
 4824#endif
 4825                phasemis = bus_phase != P_MESGIN;
 4826                if (phasemis) {
 4827#ifdef AHD_DEBUG
 4828                        if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 4829                                printf(" PHASEMIS %s\n",
 4830                                       ahd_lookup_phase_entry(bus_phase)
 4831                                                             ->phasemsg);
 4832                        }
 4833#endif
 4834                        ahd->msgin_index = 0;
 4835                        if (bus_phase == P_MESGOUT
 4836                         && (ahd->send_msg_perror != 0
 4837                          || (ahd->msgout_len != 0
 4838                           && ahd->msgout_index == 0))) {
 4839                                ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 4840                                goto reswitch;
 4841                        }
 4842                        end_session = TRUE;
 4843                        break;
 4844                }
 4845
 4846                /* Pull the byte in without acking it */
 4847                ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
 4848#ifdef AHD_DEBUG
 4849                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 4850                        printf(" byte 0x%x\n",
 4851                               ahd->msgin_buf[ahd->msgin_index]);
 4852#endif
 4853
 4854                message_done = ahd_parse_msg(ahd, &devinfo);
 4855
 4856                if (message_done) {
 4857                        /*
 4858                         * Clear our incoming message buffer in case there
 4859                         * is another message following this one.
 4860                         */
 4861                        ahd->msgin_index = 0;
 4862
 4863                        /*
 4864                         * If this message illicited a response,
 4865                         * assert ATN so the target takes us to the
 4866                         * message out phase.
 4867                         */
 4868                        if (ahd->msgout_len != 0) {
 4869#ifdef AHD_DEBUG
 4870                                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
 4871                                        ahd_print_devinfo(ahd, &devinfo);
 4872                                        printf("Asserting ATN for response\n");
 4873                                }
 4874#endif
 4875                                ahd_assert_atn(ahd);
 4876                        }
 4877                } else 
 4878                        ahd->msgin_index++;
 4879
 4880                if (message_done == MSGLOOP_TERMINATED) {
 4881                        end_session = TRUE;
 4882                } else {
 4883                        /* Ack the byte */
 4884                        ahd_outb(ahd, CLRSINT1, CLRREQINIT);
 4885                        ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
 4886                }
 4887                break;
 4888        }
 4889        case MSG_TYPE_TARGET_MSGIN:
 4890        {
 4891                int msgdone;
 4892                int msgout_request;
 4893
 4894                /*
 4895                 * By default, the message loop will continue.
 4896                 */
 4897                ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
 4898
 4899                if (ahd->msgout_len == 0)
 4900                        panic("Target MSGIN with no active message");
 4901
 4902                /*
 4903                 * If we interrupted a mesgout session, the initiator
 4904                 * will not know this until our first REQ.  So, we
 4905                 * only honor mesgout requests after we've sent our
 4906                 * first byte.
 4907                 */
 4908                if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
 4909                 && ahd->msgout_index > 0)
 4910                        msgout_request = TRUE;
 4911                else
 4912                        msgout_request = FALSE;
 4913
 4914                if (msgout_request) {
 4915
 4916                        /*
 4917                         * Change gears and see if
 4918                         * this messages is of interest to
 4919                         * us or should be passed back to
 4920                         * the sequencer.
 4921                         */
 4922                        ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
 4923                        ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
 4924                        ahd->msgin_index = 0;
 4925                        /* Dummy read to REQ for first byte */
 4926                        ahd_inb(ahd, SCSIDAT);
 4927                        ahd_outb(ahd, SXFRCTL0,
 4928                                 ahd_inb(ahd, SXFRCTL0) | SPIOEN);
 4929                        break;
 4930                }
 4931
 4932                msgdone = ahd->msgout_index == ahd->msgout_len;
 4933                if (msgdone) {
 4934                        ahd_outb(ahd, SXFRCTL0,
 4935                                 ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
 4936                        end_session = TRUE;
 4937                        break;
 4938                }
 4939
 4940                /*
 4941                 * Present the next byte on the bus.
 4942                 */
 4943                ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
 4944                ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
 4945                break;
 4946        }
 4947        case MSG_TYPE_TARGET_MSGOUT:
 4948        {
 4949                int lastbyte;
 4950                int msgdone;
 4951
 4952                /*
 4953                 * By default, the message loop will continue.
 4954                 */
 4955                ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
 4956
 4957                /*
 4958                 * The initiator signals that this is
 4959                 * the last byte by dropping ATN.
 4960                 */
 4961                lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
 4962
 4963                /*
 4964                 * Read the latched byte, but turn off SPIOEN first
 4965                 * so that we don't inadvertently cause a REQ for the
 4966                 * next byte.
 4967                 */
 4968                ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
 4969                ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
 4970                msgdone = ahd_parse_msg(ahd, &devinfo);
 4971                if (msgdone == MSGLOOP_TERMINATED) {
 4972                        /*
 4973                         * The message is *really* done in that it caused
 4974                         * us to go to bus free.  The sequencer has already
 4975                         * been reset at this point, so pull the ejection
 4976                         * handle.
 4977                         */
 4978                        return;
 4979                }
 4980                
 4981                ahd->msgin_index++;
 4982
 4983                /*
 4984                 * XXX Read spec about initiator dropping ATN too soon
 4985                 *     and use msgdone to detect it.
 4986                 */
 4987                if (msgdone == MSGLOOP_MSGCOMPLETE) {
 4988                        ahd->msgin_index = 0;
 4989
 4990                        /*
 4991                         * If this message illicited a response, transition
 4992                         * to the Message in phase and send it.
 4993                         */
 4994                        if (ahd->msgout_len != 0) {
 4995                                ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
 4996                                ahd_outb(ahd, SXFRCTL0,
 4997                                         ahd_inb(ahd, SXFRCTL0) | SPIOEN);
 4998                                ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
 4999                                ahd->msgin_index = 0;
 5000                                break;
 5001                        }
 5002                }
 5003
 5004                if (lastbyte)
 5005                        end_session = TRUE;
 5006                else {
 5007                        /* Ask for the next byte. */
 5008                        ahd_outb(ahd, SXFRCTL0,
 5009                                 ahd_inb(ahd, SXFRCTL0) | SPIOEN);
 5010                }
 5011
 5012                break;
 5013        }
 5014        default:
 5015                panic("Unknown REQINIT message type");
 5016        }
 5017
 5018        if (end_session) {
 5019                if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
 5020                        printf("%s: Returning to Idle Loop\n",
 5021                               ahd_name(ahd));
 5022                        ahd_clear_msg_state(ahd);
 5023
 5024                        /*
 5025                         * Perform the equivalent of a clear_target_state.
 5026                         */
 5027                        ahd_outb(ahd, LASTPHASE, P_BUSFREE);
 5028                        ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
 5029                        ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
 5030                } else {
 5031                        ahd_clear_msg_state(ahd);
 5032                        ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
 5033                }
 5034        }
 5035}
 5036
 5037/*
 5038 * See if we sent a particular extended message to the target.
 5039 * If "full" is true, return true only if the target saw the full
 5040 * message.  If "full" is false, return true if the target saw at
 5041 * least the first byte of the message.
 5042 */
 5043static int
 5044ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
 5045{
 5046        int found;
 5047        u_int index;
 5048
 5049        found = FALSE;
 5050        index = 0;
 5051
 5052        while (index < ahd->msgout_len) {
 5053                if (ahd->msgout_buf[index] == MSG_EXTENDED) {
 5054                        u_int end_index;
 5055
 5056                        end_index = index + 1 + ahd->msgout_buf[index + 1];
 5057                        if (ahd->msgout_buf[index+2] == msgval
 5058                         && type == AHDMSG_EXT) {
 5059
 5060                                if (full) {
 5061                                        if (ahd->msgout_index > end_index)
 5062                                                found = TRUE;
 5063                                } else if (ahd->msgout_index > index)
 5064                                        found = TRUE;
 5065                        }
 5066                        index = end_index;
 5067                } else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
 5068                        && ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
 5069
 5070                        /* Skip tag type and tag id or residue param*/
 5071                        index += 2;
 5072                } else {
 5073                        /* Single byte message */
 5074                        if (type == AHDMSG_1B
 5075                         && ahd->msgout_index > index
 5076                         && (ahd->msgout_buf[index] == msgval
 5077                          || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
 5078                           && msgval == MSG_IDENTIFYFLAG)))
 5079                                found = TRUE;
 5080                        index++;
 5081                }
 5082
 5083                if (found)
 5084                        break;
 5085        }
 5086        return (found);
 5087}
 5088
 5089/*
 5090 * Wait for a complete incoming message, parse it, and respond accordingly.
 5091 */
 5092static int
 5093ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 5094{
 5095        struct        ahd_initiator_tinfo *tinfo;
 5096        struct        ahd_tmode_tstate *tstate;
 5097        int        reject;
 5098        int        done;
 5099        int        response;
 5100
 5101        done = MSGLOOP_IN_PROG;
 5102        response = FALSE;
 5103        reject = FALSE;
 5104        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
 5105                                    devinfo->target, &tstate);
 5106
 5107        /*
 5108         * Parse as much of the message as is available,
 5109         * rejecting it if we don't support it.  When
 5110         * the entire message is available and has been
 5111         * handled, return MSGLOOP_MSGCOMPLETE, indicating
 5112         * that we have parsed an entire message.
 5113         *
 5114         * In the case of extended messages, we accept the length
 5115         * byte outright and perform more checking once we know the
 5116         * extended message type.
 5117         */
 5118        switch (ahd->msgin_buf[0]) {
 5119        case MSG_DISCONNECT:
 5120        case MSG_SAVEDATAPOINTER:
 5121        case MSG_CMDCOMPLETE:
 5122        case MSG_RESTOREPOINTERS:
 5123        case MSG_IGN_WIDE_RESIDUE:
 5124                /*
 5125                 * End our message loop as these are messages
 5126                 * the sequencer handles on its own.
 5127                 */
 5128                done = MSGLOOP_TERMINATED;
 5129                break;
 5130        case MSG_MESSAGE_REJECT:
 5131                response = ahd_handle_msg_reject(ahd, devinfo);
 5132                /* FALLTHROUGH */
 5133        case MSG_NOOP:
 5134                done = MSGLOOP_MSGCOMPLETE;
 5135                break;
 5136        case MSG_EXTENDED:
 5137        {
 5138                /* Wait for enough of the message to begin validation */
 5139                if (ahd->msgin_index < 2)
 5140                        break;
 5141                switch (ahd->msgin_buf[2]) {
 5142                case MSG_EXT_SDTR:
 5143                {
 5144                        u_int         period;
 5145                        u_int         ppr_options;
 5146                        u_int         offset;
 5147                        u_int         saved_offset;
 5148                        
 5149                        if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
 5150                                reject = TRUE;
 5151                                break;
 5152                        }
 5153
 5154                        /*
 5155                         * Wait until we have both args before validating
 5156                         * and acting on this message.
 5157                         *
 5158                         * Add one to MSG_EXT_SDTR_LEN to account for
 5159                         * the extended message preamble.
 5160                         */
 5161                        if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
 5162                                break;
 5163
 5164                        period = ahd->msgin_buf[3];
 5165                        ppr_options = 0;
 5166                        saved_offset = offset = ahd->msgin_buf[4];
 5167                        ahd_devlimited_syncrate(ahd, tinfo, &period,
 5168                                                &ppr_options, devinfo->role);
 5169                        ahd_validate_offset(ahd, tinfo, period, &offset,
 5170                                            tinfo->curr.width, devinfo->role);
 5171                        if (bootverbose) {
 5172                                printf("(%s:%c:%d:%d): Received "
 5173                                       "SDTR period %x, offset %x\n\t"
 5174                                       "Filtered to period %x, offset %x\n",
 5175                                       ahd_name(ahd), devinfo->channel,
 5176                                       devinfo->target, devinfo->lun,
 5177                                       ahd->msgin_buf[3], saved_offset,
 5178                                       period, offset);
 5179                        }
 5180                        ahd_set_syncrate(ahd, devinfo, period,
 5181                                         offset, ppr_options,
 5182                                         AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 5183                                         /*paused*/TRUE);
 5184
 5185                        /*
 5186                         * See if we initiated Sync Negotiation
 5187                         * and didn't have to fall down to async
 5188                         * transfers.
 5189                         */
 5190                        if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
 5191                                /* We started it */
 5192                                if (saved_offset != offset) {
 5193                                        /* Went too low - force async */
 5194                                        reject = TRUE;
 5195                                }
 5196                        } else {
 5197                                /*
 5198                                 * Send our own SDTR in reply
 5199                                 */
 5200                                if (bootverbose
 5201                                 && devinfo->role == ROLE_INITIATOR) {
 5202                                        printf("(%s:%c:%d:%d): Target "
 5203                                               "Initiated SDTR\n",
 5204                                               ahd_name(ahd), devinfo->channel,
 5205                                               devinfo->target, devinfo->lun);
 5206                                }
 5207                                ahd->msgout_index = 0;
 5208                                ahd->msgout_len = 0;
 5209                                ahd_construct_sdtr(ahd, devinfo,
 5210                                                   period, offset);
 5211                                ahd->msgout_index = 0;
 5212                                response = TRUE;
 5213                        }
 5214                        done = MSGLOOP_MSGCOMPLETE;
 5215                        break;
 5216                }
 5217                case MSG_EXT_WDTR:
 5218                {
 5219                        u_int bus_width;
 5220                        u_int saved_width;
 5221                        u_int sending_reply;
 5222
 5223                        sending_reply = FALSE;
 5224                        if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
 5225                                reject = TRUE;
 5226                                break;
 5227                        }
 5228
 5229                        /*
 5230                         * Wait until we have our arg before validating
 5231                         * and acting on this message.
 5232                         *
 5233                         * Add one to MSG_EXT_WDTR_LEN to account for
 5234                         * the extended message preamble.
 5235                         */
 5236                        if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
 5237                                break;
 5238
 5239                        bus_width = ahd->msgin_buf[3];
 5240                        saved_width = bus_width;
 5241                        ahd_validate_width(ahd, tinfo, &bus_width,
 5242                                           devinfo->role);
 5243                        if (bootverbose) {
 5244                                printf("(%s:%c:%d:%d): Received WDTR "
 5245                                       "%x filtered to %x\n",
 5246                                       ahd_name(ahd), devinfo->channel,
 5247                                       devinfo->target, devinfo->lun,
 5248                                       saved_width, bus_width);
 5249                        }
 5250
 5251                        if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
 5252                                /*
 5253                                 * Don't send a WDTR back to the
 5254                                 * target, since we asked first.
 5255                                 * If the width went higher than our
 5256                                 * request, reject it.
 5257                                 */
 5258                                if (saved_width > bus_width) {
 5259                                        reject = TRUE;
 5260                                        printf("(%s:%c:%d:%d): requested %dBit "
 5261                                               "transfers.  Rejecting...\n",
 5262                                               ahd_name(ahd), devinfo->channel,
 5263                                               devinfo->target, devinfo->lun,
 5264                                               8 * (0x01 << bus_width));
 5265                                        bus_width = 0;
 5266                                }
 5267                        } else {
 5268                                /*
 5269                                 * Send our own WDTR in reply
 5270                                 */
 5271                                if (bootverbose
 5272                                 && devinfo->role == ROLE_INITIATOR) {
 5273                                        printf("(%s:%c:%d:%d): Target "
 5274                                               "Initiated WDTR\n",
 5275                                               ahd_name(ahd), devinfo->channel,
 5276                                               devinfo->target, devinfo->lun);
 5277                                }
 5278                                ahd->msgout_index = 0;
 5279                                ahd->msgout_len = 0;
 5280                                ahd_construct_wdtr(ahd, devinfo, bus_width);
 5281                                ahd->msgout_index = 0;
 5282                                response = TRUE;
 5283                                sending_reply = TRUE;
 5284                        }
 5285                        /*
 5286                         * After a wide message, we are async, but
 5287                         * some devices don't seem to honor this portion
 5288                         * of the spec.  Force a renegotiation of the
 5289                         * sync component of our transfer agreement even
 5290                         * if our goal is async.  By updating our width
 5291                         * after forcing the negotiation, we avoid
 5292                         * renegotiating for width.
 5293                         */
 5294                        ahd_update_neg_request(ahd, devinfo, tstate,
 5295                                               tinfo, AHD_NEG_ALWAYS);
 5296                        ahd_set_width(ahd, devinfo, bus_width,
 5297                                      AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 5298                                      /*paused*/TRUE);
 5299                        if (sending_reply == FALSE && reject == FALSE) {
 5300
 5301                                /*
 5302                                 * We will always have an SDTR to send.
 5303                                 */
 5304                                ahd->msgout_index = 0;
 5305                                ahd->msgout_len = 0;
 5306                                ahd_build_transfer_msg(ahd, devinfo);
 5307                                ahd->msgout_index = 0;
 5308                                response = TRUE;
 5309                        }
 5310                        done = MSGLOOP_MSGCOMPLETE;
 5311                        break;
 5312                }
 5313                case MSG_EXT_PPR:
 5314                {
 5315                        u_int        period;
 5316                        u_int        offset;
 5317                        u_int        bus_width;
 5318                        u_int        ppr_options;
 5319                        u_int        saved_width;
 5320                        u_int        saved_offset;
 5321                        u_int        saved_ppr_options;
 5322
 5323                        if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
 5324                                reject = TRUE;
 5325                                break;
 5326                        }
 5327
 5328                        /*
 5329                         * Wait until we have all args before validating
 5330                         * and acting on this message.
 5331                         *
 5332                         * Add one to MSG_EXT_PPR_LEN to account for
 5333                         * the extended message preamble.
 5334                         */
 5335                        if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
 5336                                break;
 5337
 5338                        period = ahd->msgin_buf[3];
 5339                        offset = ahd->msgin_buf[5];
 5340                        bus_width = ahd->msgin_buf[6];
 5341                        saved_width = bus_width;
 5342                        ppr_options = ahd->msgin_buf[7];
 5343                        /*
 5344                         * According to the spec, a DT only
 5345                         * period factor with no DT option
 5346                         * set implies async.
 5347                         */
 5348                        if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
 5349                         && period <= 9)
 5350                                offset = 0;
 5351                        saved_ppr_options = ppr_options;
 5352                        saved_offset = offset;
 5353
 5354                        /*
 5355                         * Transfer options are only available if we
 5356                         * are negotiating wide.
 5357                         */
 5358                        if (bus_width == 0)
 5359                                ppr_options &= MSG_EXT_PPR_QAS_REQ;
 5360
 5361                        ahd_validate_width(ahd, tinfo, &bus_width,
 5362                                           devinfo->role);
 5363                        ahd_devlimited_syncrate(ahd, tinfo, &period,
 5364                                                &ppr_options, devinfo->role);
 5365                        ahd_validate_offset(ahd, tinfo, period, &offset,
 5366                                            bus_width, devinfo->role);
 5367
 5368                        if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
 5369                                /*
 5370                                 * If we are unable to do any of the
 5371                                 * requested options (we went too low),
 5372                                 * then we'll have to reject the message.
 5373                                 */
 5374                                if (saved_width > bus_width
 5375                                 || saved_offset != offset
 5376                                 || saved_ppr_options != ppr_options) {
 5377                                        reject = TRUE;
 5378                                        period = 0;
 5379                                        offset = 0;
 5380                                        bus_width = 0;
 5381                                        ppr_options = 0;
 5382                                }
 5383                        } else {
 5384                                if (devinfo->role != ROLE_TARGET)
 5385                                        printf("(%s:%c:%d:%d): Target "
 5386                                               "Initiated PPR\n",
 5387                                               ahd_name(ahd), devinfo->channel,
 5388                                               devinfo->target, devinfo->lun);
 5389                                else
 5390                                        printf("(%s:%c:%d:%d): Initiator "
 5391                                               "Initiated PPR\n",
 5392                                               ahd_name(ahd), devinfo->channel,
 5393                                               devinfo->target, devinfo->lun);
 5394                                ahd->msgout_index = 0;
 5395                                ahd->msgout_len = 0;
 5396                                ahd_construct_ppr(ahd, devinfo, period, offset,
 5397                                                  bus_width, ppr_options);
 5398                                ahd->msgout_index = 0;
 5399                                response = TRUE;
 5400                        }
 5401                        if (bootverbose) {
 5402                                printf("(%s:%c:%d:%d): Received PPR width %x, "
 5403                                       "period %x, offset %x,options %x\n"
 5404                                       "\tFiltered to width %x, period %x, "
 5405                                       "offset %x, options %x\n",
 5406                                       ahd_name(ahd), devinfo->channel,
 5407                                       devinfo->target, devinfo->lun,
 5408                                       saved_width, ahd->msgin_buf[3],
 5409                                       saved_offset, saved_ppr_options,
 5410                                       bus_width, period, offset, ppr_options);
 5411                        }
 5412                        ahd_set_width(ahd, devinfo, bus_width,
 5413                                      AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 5414                                      /*paused*/TRUE);
 5415                        ahd_set_syncrate(ahd, devinfo, period,
 5416                                         offset, ppr_options,
 5417                                         AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 5418                                         /*paused*/TRUE);
 5419
 5420                        done = MSGLOOP_MSGCOMPLETE;
 5421                        break;
 5422                }
 5423                default:
 5424                        /* Unknown extended message.  Reject it. */
 5425                        reject = TRUE;
 5426                        break;
 5427                }
 5428                break;
 5429        }
 5430#ifdef AHD_TARGET_MODE
 5431        case MSG_BUS_DEV_RESET:
 5432                ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
 5433                                    CAM_BDR_SENT,
 5434                                    "Bus Device Reset Received",
 5435                                    /*verbose_level*/0);
 5436                ahd_restart(ahd);
 5437                done = MSGLOOP_TERMINATED;
 5438                break;
 5439        case MSG_ABORT_TAG:
 5440        case MSG_ABORT:
 5441        case MSG_CLEAR_QUEUE:
 5442        {
 5443                int tag;
 5444
 5445                /* Target mode messages */
 5446                if (devinfo->role != ROLE_TARGET) {
 5447                        reject = TRUE;
 5448                        break;
 5449                }
 5450                tag = SCB_LIST_NULL;
 5451                if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
 5452                        tag = ahd_inb(ahd, INITIATOR_TAG);
 5453                ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
 5454                               devinfo->lun, tag, ROLE_TARGET,
 5455                               CAM_REQ_ABORTED);
 5456
 5457                tstate = ahd->enabled_targets[devinfo->our_scsiid];
 5458                if (tstate != NULL) {
 5459                        struct ahd_tmode_lstate* lstate;
 5460
 5461                        lstate = tstate->enabled_luns[devinfo->lun];
 5462                        if (lstate != NULL) {
 5463                                ahd_queue_lstate_event(ahd, lstate,
 5464                                                       devinfo->our_scsiid,
 5465                                                       ahd->msgin_buf[0],
 5466                                                       /*arg*/tag);
 5467                                ahd_send_lstate_events(ahd, lstate);
 5468                        }
 5469                }
 5470                ahd_restart(ahd);
 5471                done = MSGLOOP_TERMINATED;
 5472                break;
 5473        }
 5474#endif
 5475        case MSG_QAS_REQUEST:
 5476#ifdef AHD_DEBUG
 5477                if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
 5478                        printf("%s: QAS request.  SCSISIGI == 0x%x\n",
 5479                               ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
 5480#endif
 5481                ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
 5482                /* FALLTHROUGH */
 5483        case MSG_TERM_IO_PROC:
 5484        default:
 5485                reject = TRUE;
 5486                break;
 5487        }
 5488
 5489        if (reject) {
 5490                /*
 5491                 * Setup to reject the message.
 5492                 */
 5493                ahd->msgout_index = 0;
 5494                ahd->msgout_len = 1;
 5495                ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
 5496                done = MSGLOOP_MSGCOMPLETE;
 5497                response = TRUE;
 5498        }
 5499
 5500        if (done != MSGLOOP_IN_PROG && !response)
 5501                /* Clear the outgoing message buffer */
 5502                ahd->msgout_len = 0;
 5503
 5504        return (done);
 5505}
 5506
 5507/*
 5508 * Process a message reject message.
 5509 */
 5510static int
 5511ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 5512{
 5513        /*
 5514         * What we care about here is if we had an
 5515         * outstanding SDTR or WDTR message for this
 5516         * target.  If we did, this is a signal that
 5517         * the target is refusing negotiation.
 5518         */
 5519        struct scb *scb;
 5520        struct ahd_initiator_tinfo *tinfo;
 5521        struct ahd_tmode_tstate *tstate;
 5522        u_int scb_index;
 5523        u_int last_msg;
 5524        int   response = 0;
 5525
 5526        scb_index = ahd_get_scbptr(ahd);
 5527        scb = ahd_lookup_scb(ahd, scb_index);
 5528        tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
 5529                                    devinfo->our_scsiid,
 5530                                    devinfo->target, &tstate);
 5531        /* Might be necessary */
 5532        last_msg = ahd_inb(ahd, LAST_MSG);
 5533
 5534        if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
 5535                if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
 5536                 && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
 5537                        /*
 5538                         * Target may not like our SPI-4 PPR Options.
 5539                         * Attempt to negotiate 80MHz which will turn
 5540                         * off these options.
 5541                         */
 5542                        if (bootverbose) {
 5543                                printf("(%s:%c:%d:%d): PPR Rejected. "
 5544                                       "Trying simple U160 PPR\n",
 5545                                       ahd_name(ahd), devinfo->channel,
 5546                                       devinfo->target, devinfo->lun);
 5547                        }
 5548                        tinfo->goal.period = AHD_SYNCRATE_DT;
 5549                        tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
 5550                                                |  MSG_EXT_PPR_QAS_REQ
 5551                                                |  MSG_EXT_PPR_DT_REQ;
 5552                } else {
 5553                        /*
 5554                         * Target does not support the PPR message.
 5555                         * Attempt to negotiate SPI-2 style.
 5556                         */
 5557                        if (bootverbose) {
 5558                                printf("(%s:%c:%d:%d): PPR Rejected. "
 5559                                       "Trying WDTR/SDTR\n",
 5560                                       ahd_name(ahd), devinfo->channel,
 5561                                       devinfo->target, devinfo->lun);
 5562                        }
 5563                        tinfo->goal.ppr_options = 0;
 5564                        tinfo->curr.transport_version = 2;
 5565                        tinfo->goal.transport_version = 2;
 5566                }
 5567                ahd->msgout_index = 0;
 5568                ahd->msgout_len = 0;
 5569                ahd_build_transfer_msg(ahd, devinfo);
 5570                ahd->msgout_index = 0;
 5571                response = 1;
 5572        } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
 5573
 5574                /* note 8bit xfers */
 5575                printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
 5576                       "8bit transfers\n", ahd_name(ahd),
 5577                       devinfo->channel, devinfo->target, devinfo->lun);
 5578                ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
 5579                              AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 5580                              /*paused*/TRUE);
 5581                /*
 5582                 * No need to clear the sync rate.  If the target
 5583                 * did not accept the command, our syncrate is
 5584                 * unaffected.  If the target started the negotiation,
 5585                 * but rejected our response, we already cleared the
 5586                 * sync rate before sending our WDTR.
 5587                 */
 5588                if (tinfo->goal.offset != tinfo->curr.offset) {
 5589
 5590                        /* Start the sync negotiation */
 5591                        ahd->msgout_index = 0;
 5592                        ahd->msgout_len = 0;
 5593                        ahd_build_transfer_msg(ahd, devinfo);
 5594                        ahd->msgout_index = 0;
 5595                        response = 1;
 5596                }
 5597        } else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
 5598                /* note asynch xfers and clear flag */
 5599                ahd_set_syncrate(ahd, devinfo, /*period*/0,
 5600                                 /*offset*/0, /*ppr_options*/0,
 5601                                 AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
 5602                                 /*paused*/TRUE);
 5603                printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
 5604                       "Using asynchronous transfers\n",
 5605                       ahd_name(ahd), devinfo->channel,
 5606                       devinfo->target, devinfo->lun);
 5607        } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
 5608                int tag_type;
 5609                int mask;
 5610
 5611                tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
 5612
 5613                if (tag_type == MSG_SIMPLE_TASK) {
 5614                        printf("(%s:%c:%d:%d): refuses tagged commands.  "
 5615                               "Performing non-tagged I/O\n", ahd_name(ahd),
 5616                               devinfo->channel, devinfo->target, devinfo->lun);
 5617                        ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_NONE);
 5618                        mask = ~0x23;
 5619                } else {
 5620                        printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
 5621                               "Performing simple queue tagged I/O only\n",
 5622                               ahd_name(ahd), devinfo->channel, devinfo->target,
 5623                               devinfo->lun, tag_type == MSG_ORDERED_TASK
 5624                               ? "ordered" : "head of queue");
 5625                        ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_BASIC);
 5626                        mask = ~0x03;
 5627                }
 5628
 5629                /*
 5630                 * Resend the identify for this CCB as the target
 5631                 * may believe that the selection is invalid otherwise.
 5632                 */
 5633                ahd_outb(ahd, SCB_CONTROL,
 5634                         ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
 5635                 scb->hscb->control &= mask;
 5636                ahd_set_transaction_tag(scb, /*enabled*/FALSE,
 5637                                        /*type*/MSG_SIMPLE_TASK);
 5638                ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
 5639                ahd_assert_atn(ahd);
 5640                ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
 5641                             SCB_GET_TAG(scb));
 5642
 5643                /*
 5644                 * Requeue all tagged commands for this target
 5645                 * currently in our posession so they can be
 5646                 * converted to untagged commands.
 5647                 */
 5648                ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
 5649                                   SCB_GET_CHANNEL(ahd, scb),
 5650                                   SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
 5651                                   ROLE_INITIATOR, CAM_REQUEUE_REQ,
 5652                                   SEARCH_COMPLETE);
 5653        } else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
 5654                /*
 5655                 * Most likely the device believes that we had
 5656                 * previously negotiated packetized.
 5657                 */
 5658                ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
 5659                               |  MSG_FLAG_IU_REQ_CHANGED;
 5660
 5661                ahd_force_renegotiation(ahd, devinfo);
 5662                ahd->msgout_index = 0;
 5663                ahd->msgout_len = 0;
 5664                ahd_build_transfer_msg(ahd, devinfo);
 5665                ahd->msgout_index = 0;
 5666                response = 1;
 5667        } else {
 5668                /*
 5669                 * Otherwise, we ignore it.
 5670                 */
 5671                printf("%s:%c:%d: Message reject for %x -- ignored\n",
 5672                       ahd_name(ahd), devinfo->channel, devinfo->target,
 5673                       last_msg);
 5674        }
 5675        return (response);
 5676}
 5677
 5678/*
 5679 * Process an ingnore wide residue message.
 5680 */
 5681static void
 5682ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
 5683{
 5684        u_int scb_index;
 5685        struct scb *scb;
 5686
 5687        scb_index = ahd_get_scbptr(ahd);
 5688        scb = ahd_lookup_scb(ahd, scb_index);
 5689        /*
 5690         * XXX Actually check data direction in the sequencer?
 5691         * Perhaps add datadir to some spare bits in the hscb?
 5692         */
 5693        if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
 5694         || ahd_get_transfer_dir(scb) != CAM_DIR_IN) {
 5695                /*
 5696                 * Ignore the message if we haven't
 5697                 * seen an appropriate data phase yet.
 5698                 */
 5699        } else {
 5700                /*
 5701                 * If the residual occurred on the last
 5702                 * transfer and the transfer request was
 5703                 * expected to end on an odd count, do
 5704                 * nothing.  Otherwise, subtract a byte
 5705                 * and update the residual count accordingly.
 5706                 */
 5707                uint32_t sgptr;
 5708
 5709                sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
 5710                if ((sgptr & SG_LIST_NULL) != 0
 5711                 && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
 5712                     & SCB_XFERLEN_ODD) != 0) {
 5713                        /*
 5714                         * If the residual occurred on the last
 5715                         * transfer and the transfer request was
 5716                         * expected to end on an odd count, do
 5717                         * nothing.
 5718                         */
 5719                } else {
 5720                        uint32_t data_cnt;
 5721                        uint64_t data_addr;
 5722                        uint32_t sglen;
 5723
 5724                        /* Pull in the rest of the sgptr */
 5725                        sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 5726                        data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
 5727                        if ((sgptr & SG_LIST_NULL) != 0) {
 5728                                /*
 5729                                 * The residual data count is not updated
 5730                                 * for the command run to completion case.
 5731                                 * Explicitly zero the count.
 5732                                 */
 5733                                data_cnt &= ~AHD_SG_LEN_MASK;
 5734                        }
 5735                        data_addr = ahd_inq(ahd, SHADDR);
 5736                        data_cnt += 1;
 5737                        data_addr -= 1;
 5738                        sgptr &= SG_PTR_MASK;
 5739                        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 5740                                struct ahd_dma64_seg *sg;
 5741
 5742                                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 5743
 5744                                /*
 5745                                 * The residual sg ptr points to the next S/G
 5746                                 * to load so we must go back one.
 5747                                 */
 5748                                sg--;
 5749                                sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
 5750                                if (sg != scb->sg_list
 5751                                 && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
 5752
 5753                                        sg--;
 5754                                        sglen = ahd_le32toh(sg->len);
 5755                                        /*
 5756                                         * Preserve High Address and SG_LIST
 5757                                         * bits while setting the count to 1.
 5758                                         */
 5759                                        data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
 5760                                        data_addr = ahd_le64toh(sg->addr)
 5761                                                  + (sglen & AHD_SG_LEN_MASK)
 5762                                                  - 1;
 5763
 5764                                        /*
 5765                                         * Increment sg so it points to the
 5766                                         * "next" sg.
 5767                                         */
 5768                                        sg++;
 5769                                        sgptr = ahd_sg_virt_to_bus(ahd, scb,
 5770                                                                   sg);
 5771                                }
 5772                        } else {
 5773                                struct ahd_dma_seg *sg;
 5774
 5775                                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 5776
 5777                                /*
 5778                                 * The residual sg ptr points to the next S/G
 5779                                 * to load so we must go back one.
 5780                                 */
 5781                                sg--;
 5782                                sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
 5783                                if (sg != scb->sg_list
 5784                                 && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
 5785
 5786                                        sg--;
 5787                                        sglen = ahd_le32toh(sg->len);
 5788                                        /*
 5789                                         * Preserve High Address and SG_LIST
 5790                                         * bits while setting the count to 1.
 5791                                         */
 5792                                        data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
 5793                                        data_addr = ahd_le32toh(sg->addr)
 5794                                                  + (sglen & AHD_SG_LEN_MASK)
 5795                                                  - 1;
 5796
 5797                                        /*
 5798                                         * Increment sg so it points to the
 5799                                         * "next" sg.
 5800                                         */
 5801                                        sg++;
 5802                                        sgptr = ahd_sg_virt_to_bus(ahd, scb,
 5803                                                                  sg);
 5804                                }
 5805                        }
 5806                        /*
 5807                         * Toggle the "oddness" of the transfer length
 5808                         * to handle this mid-transfer ignore wide
 5809                         * residue.  This ensures that the oddness is
 5810                         * correct for subsequent data transfers.
 5811                         */
 5812                        ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
 5813                            ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
 5814                            ^ SCB_XFERLEN_ODD);
 5815
 5816                        ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
 5817                        ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
 5818                        /*
 5819                         * The FIFO's pointers will be updated if/when the
 5820                         * sequencer re-enters a data phase.
 5821                         */
 5822                }
 5823        }
 5824}
 5825
 5826
 5827/*
 5828 * Reinitialize the data pointers for the active transfer
 5829 * based on its current residual.
 5830 */
 5831static void
 5832ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
 5833{
 5834        struct                 scb *scb;
 5835        ahd_mode_state         saved_modes;
 5836        u_int                 scb_index;
 5837        u_int                 wait;
 5838        uint32_t         sgptr;
 5839        uint32_t         resid;
 5840        uint64_t         dataptr;
 5841
 5842        AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
 5843                         AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
 5844                         
 5845        scb_index = ahd_get_scbptr(ahd);
 5846        scb = ahd_lookup_scb(ahd, scb_index);
 5847
 5848        /*
 5849         * Release and reacquire the FIFO so we
 5850         * have a clean slate.
 5851         */
 5852        ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
 5853        wait = 1000;
 5854        while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
 5855                ahd_delay(100);
 5856        if (wait == 0) {
 5857                ahd_print_path(ahd, scb);
 5858                printf("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
 5859                ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
 5860        }
 5861        saved_modes = ahd_save_modes(ahd);
 5862        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 5863        ahd_outb(ahd, DFFSTAT,
 5864                 ahd_inb(ahd, DFFSTAT)
 5865                | (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
 5866
 5867        /*
 5868         * Determine initial values for data_addr and data_cnt
 5869         * for resuming the data phase.
 5870         */
 5871        sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
 5872        sgptr &= SG_PTR_MASK;
 5873
 5874        resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
 5875              | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
 5876              | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
 5877
 5878        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
 5879                struct ahd_dma64_seg *sg;
 5880
 5881                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 5882
 5883                /* The residual sg_ptr always points to the next sg */
 5884                sg--;
 5885
 5886                dataptr = ahd_le64toh(sg->addr)
 5887                        + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
 5888                        - resid;
 5889                ahd_outl(ahd, HADDR + 4, dataptr >> 32);
 5890        } else {
 5891                struct         ahd_dma_seg *sg;
 5892
 5893                sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
 5894
 5895                /* The residual sg_ptr always points to the next sg */
 5896                sg--;
 5897
 5898                dataptr = ahd_le32toh(sg->addr)
 5899                        + (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
 5900                        - resid;
 5901                ahd_outb(ahd, HADDR + 4,
 5902                         (ahd_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
 5903        }
 5904        ahd_outl(ahd, HADDR, dataptr);
 5905        ahd_outb(ahd, HCNT + 2, resid >> 16);
 5906        ahd_outb(ahd, HCNT + 1, resid >> 8);
 5907        ahd_outb(ahd, HCNT, resid);
 5908}
 5909
 5910/*
 5911 * Handle the effects of issuing a bus device reset message.
 5912 */
 5913static void
 5914ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 5915                    u_int lun, cam_status status, char *message,
 5916                    int verbose_level)
 5917{
 5918#ifdef AHD_TARGET_MODE
 5919        struct ahd_tmode_tstate* tstate;
 5920#endif
 5921        int found;
 5922
 5923        found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
 5924                               lun, SCB_LIST_NULL, devinfo->role,
 5925                               status);
 5926
 5927#ifdef AHD_TARGET_MODE
 5928        /*
 5929         * Send an immediate notify ccb to all target mord peripheral
 5930         * drivers affected by this action.
 5931         */
 5932        tstate = ahd->enabled_targets[devinfo->our_scsiid];
 5933        if (tstate != NULL) {
 5934                u_int cur_lun;
 5935                u_int max_lun;
 5936
 5937                if (lun != CAM_LUN_WILDCARD) {
 5938                        cur_lun = 0;
 5939                        max_lun = AHD_NUM_LUNS - 1;
 5940                } else {
 5941                        cur_lun = lun;
 5942                        max_lun = lun;
 5943                }
 5944                for (;cur_lun <= max_lun; cur_lun++) {
 5945                        struct ahd_tmode_lstate* lstate;
 5946
 5947                        lstate = tstate->enabled_luns[cur_lun];
 5948                        if (lstate == NULL)
 5949                                continue;
 5950
 5951                        ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
 5952                                               MSG_BUS_DEV_RESET, /*arg*/0);
 5953                        ahd_send_lstate_events(ahd, lstate);
 5954                }
 5955        }
 5956#endif
 5957
 5958        /*
 5959         * Go back to async/narrow transfers and renegotiate.
 5960         */
 5961        ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
 5962                      AHD_TRANS_CUR, /*paused*/TRUE);
 5963        ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
 5964                         /*ppr_options*/0, AHD_TRANS_CUR,
 5965                         /*paused*/TRUE);
 5966        
 5967        if (status != CAM_SEL_TIMEOUT)
 5968                ahd_send_async(ahd, devinfo->channel, devinfo->target,
 5969                               CAM_LUN_WILDCARD, AC_SENT_BDR);
 5970
 5971        if (message != NULL && bootverbose)
 5972                printf("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
 5973                       message, devinfo->channel, devinfo->target, found);
 5974}
 5975
 5976#ifdef AHD_TARGET_MODE
 5977static void
 5978ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
 5979                       struct scb *scb)
 5980{
 5981
 5982        /*              
 5983         * To facilitate adding multiple messages together,
 5984         * each routine should increment the index and len
 5985         * variables instead of setting them explicitly.
 5986         */             
 5987        ahd->msgout_index = 0;
 5988        ahd->msgout_len = 0;
 5989
 5990        if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
 5991                ahd_build_transfer_msg(ahd, devinfo);
 5992        else
 5993                panic("ahd_intr: AWAITING target message with no message");
 5994
 5995        ahd->msgout_index = 0;
 5996        ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
 5997}
 5998#endif
 5999/**************************** Initialization **********************************/
 6000static u_int
 6001ahd_sglist_size(struct ahd_softc *ahd)
 6002{
 6003        bus_size_t list_size;
 6004
 6005        list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
 6006        if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
 6007                list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
 6008        return (list_size);
 6009}
 6010
 6011/*
 6012 * Calculate the optimum S/G List allocation size.  S/G elements used
 6013 * for a given transaction must be physically contiguous.  Assume the
 6014 * OS will allocate full pages to us, so it doesn't make sense to request
 6015 * less than a page.
 6016 */
 6017static u_int
 6018ahd_sglist_allocsize(struct ahd_softc *ahd)
 6019{
 6020        bus_size_t sg_list_increment;
 6021        bus_size_t sg_list_size;
 6022        bus_size_t max_list_size;
 6023        bus_size_t best_list_size;
 6024
 6025        /* Start out with the minimum required for AHD_NSEG. */
 6026        sg_list_increment = ahd_sglist_size(ahd);
 6027        sg_list_size = sg_list_increment;
 6028
 6029        /* Get us as close as possible to a page in size. */
 6030        while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
 6031                sg_list_size += sg_list_increment;
 6032
 6033        /*
 6034         * Try to reduce the amount of wastage by allocating
 6035         * multiple pages.
 6036         */
 6037        best_list_size = sg_list_size;
 6038        max_list_size = roundup(sg_list_increment, PAGE_SIZE);
 6039        if (max_list_size < 4 * PAGE_SIZE)
 6040                max_list_size = 4 * PAGE_SIZE;
 6041        if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
 6042                max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
 6043        while ((sg_list_size + sg_list_increment) <= max_list_size
 6044           &&  (sg_list_size % PAGE_SIZE) != 0) {
 6045                bus_size_t new_mod;
 6046                bus_size_t best_mod;
 6047
 6048                sg_list_size += sg_list_increment;
 6049                new_mod = sg_list_size % PAGE_SIZE;
 6050                best_mod = best_list_size % PAGE_SIZE;
 6051                if (new_mod > best_mod || new_mod == 0) {
 6052                        best_list_size = sg_list_size;
 6053                }
 6054        }
 6055        return (best_list_size);
 6056}
 6057
 6058/*
 6059 * Allocate a controller structure for a new device
 6060 * and perform initial initializion.
 6061 */
 6062struct ahd_softc *
 6063ahd_alloc(void *platform_arg, char *name)
 6064{
 6065        struct  ahd_softc *ahd;
 6066
 6067#ifndef        __FreeBSD__
 6068        ahd = malloc(sizeof(*ahd), M_DEVBUF, M_NOWAIT);
 6069        if (!ahd) {
 6070                printf("aic7xxx: cannot malloc softc!\n");
 6071                free(name, M_DEVBUF);
 6072                return NULL;
 6073        }
 6074#else
 6075        ahd = device_get_softc((device_t)platform_arg);
 6076#endif
 6077        memset(ahd, 0, sizeof(*ahd));
 6078        ahd->seep_config = malloc(sizeof(*ahd->seep_config),
 6079                                  M_DEVBUF, M_NOWAIT);
 6080        if (ahd->seep_config == NULL) {
 6081#ifndef        __FreeBSD__
 6082                free(ahd, M_DEVBUF);
 6083#endif
 6084                free(name, M_DEVBUF);
 6085                return (NULL);
 6086        }
 6087        LIST_INIT(&ahd->pending_scbs);
 6088        /* We don't know our unit number until the OSM sets it */
 6089        ahd->name = name;
 6090        ahd->unit = -1;
 6091        ahd->description = NULL;
 6092        ahd->bus_description = NULL;
 6093        ahd->channel = 'A';
 6094        ahd->chip = AHD_NONE;
 6095        ahd->features = AHD_FENONE;
 6096        ahd->bugs = AHD_BUGNONE;
 6097        ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
 6098                   | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
 6099        ahd_timer_init(&ahd->reset_timer);
 6100        ahd_timer_init(&ahd->stat_timer);
 6101        ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
 6102        ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
 6103        ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
 6104        ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
 6105        ahd->int_coalescing_stop_threshold =
 6106            AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
 6107
 6108        if (ahd_platform_alloc(ahd, platform_arg) != 0) {
 6109                ahd_free(ahd);
 6110                ahd = NULL;
 6111        }
 6112#ifdef AHD_DEBUG
 6113        if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
 6114                printf("%s: scb size = 0x%x, hscb size = 0x%x\n",
 6115                       ahd_name(ahd), (u_int)sizeof(struct scb),
 6116                       (u_int)sizeof(struct hardware_scb));
 6117        }
 6118#endif
 6119        return (ahd);
 6120}
 6121
 6122int
 6123ahd_softc_init(struct ahd_softc *ahd)
 6124{
 6125
 6126        ahd->unpause = 0;
 6127        ahd->pause = PAUSE; 
 6128        return (0);
 6129}
 6130
 6131void
 6132ahd_set_unit(struct ahd_softc *ahd, int unit)
 6133{
 6134        ahd->unit = unit;
 6135}
 6136
 6137void
 6138ahd_set_name(struct ahd_softc *ahd, char *name)
 6139{
 6140        if (ahd->name != NULL)
 6141                free(ahd->name, M_DEVBUF);
 6142        ahd->name = name;
 6143}
 6144
 6145void
 6146ahd_free(struct ahd_softc *ahd)
 6147{
 6148        int i;
 6149
 6150        switch (ahd->init_level) {
 6151        default:
 6152        case 5:
 6153                ahd_shutdown(ahd);
 6154                /* FALLTHROUGH */
 6155        case 4:
 6156                ahd_dmamap_unload(ahd, ahd->shared_data_dmat,
 6157                                  ahd->shared_data_map.dmamap);
 6158                /* FALLTHROUGH */
 6159        case 3:
 6160                ahd_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
 6161                                ahd->shared_data_map.dmamap);
 6162                ahd_dmamap_destroy(ahd, ahd->shared_data_dmat,
 6163                                   ahd->shared_data_map.dmamap);
 6164                /* FALLTHROUGH */
 6165        case 2:
 6166                ahd_dma_tag_destroy(ahd, ahd->shared_data_dmat);
 6167        case 1:
 6168#ifndef __linux__
 6169                ahd_dma_tag_destroy(ahd, ahd->buffer_dmat);
 6170#endif
 6171                break;
 6172        case 0:
 6173                break;
 6174        }
 6175
 6176#ifndef __linux__
 6177        ahd_dma_tag_destroy(ahd, ahd->parent_dmat);
 6178#endif
 6179        ahd_platform_free(ahd);
 6180        ahd_fini_scbdata(ahd);
 6181        for (i = 0; i < AHD_NUM_TARGETS; i++) {
 6182                struct ahd_tmode_tstate *tstate;
 6183
 6184                tstate = ahd->enabled_targets[i];
 6185                if (tstate != NULL) {
 6186#ifdef AHD_TARGET_MODE
 6187                        int j;
 6188
 6189                        for (j = 0; j < AHD_NUM_LUNS; j++) {
 6190                                struct ahd_tmode_lstate *lstate;
 6191
 6192                                lstate = tstate->enabled_luns[j];
 6193                                if (lstate != NULL) {
 6194                                        xpt_free_path(lstate->path);
 6195                                        free(lstate, M_DEVBUF);
 6196                                }
 6197                        }
 6198#endif
 6199                        free(tstate, M_DEVBUF);
 6200                }
 6201        }
 6202#ifdef AHD_TARGET_MODE
 6203        if (ahd->black_hole != NULL) {
 6204                xpt_free_path(ahd->black_hole->path);
 6205                free(ahd->black_hole, M_DEVBUF);
 6206        }
 6207#endif
 6208        if (ahd->name != NULL)
 6209                free(ahd->name, M_DEVBUF);
 6210        if (ahd->seep_config != NULL)
 6211                free(ahd->seep_config, M_DEVBUF);
 6212        if (ahd->saved_stack != NULL)
 6213                free(ahd->saved_stack, M_DEVBUF);
 6214#ifndef __FreeBSD__
 6215        free(ahd, M_DEVBUF);
 6216#endif
 6217        return;
 6218}
 6219
 6220static void
 6221ahd_shutdown(void *arg)
 6222{
 6223        struct        ahd_softc *ahd;
 6224
 6225        ahd = (struct ahd_softc *)arg;
 6226
 6227        /*
 6228         * Stop periodic timer callbacks.
 6229         */
 6230        ahd_timer_stop(&ahd->reset_timer);
 6231        ahd_timer_stop(&ahd->stat_timer);
 6232
 6233        /* This will reset most registers to 0, but not all */
 6234        ahd_reset(ahd, /*reinit*/FALSE);
 6235}
 6236
 6237/*
 6238 * Reset the controller and record some information about it
 6239 * that is only available just after a reset.  If "reinit" is
 6240 * non-zero, this reset occured after initial configuration
 6241 * and the caller requests that the chip be fully reinitialized
 6242 * to a runable state.  Chip interrupts are *not* enabled after
 6243 * a reinitialization.  The caller must enable interrupts via
 6244 * ahd_intr_enable().
 6245 */
 6246int
 6247ahd_reset(struct ahd_softc *ahd, int reinit)
 6248{
 6249        u_int         sxfrctl1;
 6250        int         wait;
 6251        uint32_t cmd;
 6252        
 6253        /*
 6254         * Preserve the value of the SXFRCTL1 register for all channels.
 6255         * It contains settings that affect termination and we don't want
 6256         * to disturb the integrity of the bus.
 6257         */
 6258        ahd_pause(ahd);
 6259        ahd_update_modes(ahd);
 6260        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 6261        sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
 6262
 6263        cmd = ahd_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
 6264        if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
 6265                uint32_t mod_cmd;
 6266
 6267                /*
 6268                 * A4 Razor #632
 6269                 * During the assertion of CHIPRST, the chip
 6270                 * does not disable its parity logic prior to
 6271                 * the start of the reset.  This may cause a
 6272                 * parity error to be detected and thus a
 6273                 * spurious SERR or PERR assertion.  Disble
 6274                 * PERR and SERR responses during the CHIPRST.
 6275                 */
 6276                mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
 6277                ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
 6278                                     mod_cmd, /*bytes*/2);
 6279        }
 6280        ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
 6281
 6282        /*
 6283         * Ensure that the reset has finished.  We delay 1000us
 6284         * prior to reading the register to make sure the chip
 6285         * has sufficiently completed its reset to handle register
 6286         * accesses.
 6287         */
 6288        wait = 1000;
 6289        do {
 6290                ahd_delay(1000);
 6291        } while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
 6292
 6293        if (wait == 0) {
 6294                printf("%s: WARNING - Failed chip reset!  "
 6295                       "Trying to initialize anyway.\n", ahd_name(ahd));
 6296        }
 6297        ahd_outb(ahd, HCNTRL, ahd->pause);
 6298
 6299        if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
 6300                /*
 6301                 * Clear any latched PCI error status and restore
 6302                 * previous SERR and PERR response enables.
 6303                 */
 6304                ahd_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
 6305                                     0xFF, /*bytes*/1);
 6306                ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
 6307                                     cmd, /*bytes*/2);
 6308        }
 6309
 6310        /*
 6311         * Mode should be SCSI after a chip reset, but lets
 6312         * set it just to be safe.  We touch the MODE_PTR
 6313         * register directly so as to bypass the lazy update
 6314         * code in ahd_set_modes().
 6315         */
 6316        ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 6317        ahd_outb(ahd, MODE_PTR,
 6318                 ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
 6319
 6320        /*
 6321         * Restore SXFRCTL1.
 6322         *
 6323         * We must always initialize STPWEN to 1 before we
 6324         * restore the saved values.  STPWEN is initialized
 6325         * to a tri-state condition which can only be cleared
 6326         * by turning it on.
 6327         */
 6328        ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
 6329        ahd_outb(ahd, SXFRCTL1, sxfrctl1);
 6330
 6331        /* Determine chip configuration */
 6332        ahd->features &= ~AHD_WIDE;
 6333        if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
 6334                ahd->features |= AHD_WIDE;
 6335
 6336        /*
 6337         * If a recovery action has forced a chip reset,
 6338         * re-initialize the chip to our liking.
 6339         */
 6340        if (reinit != 0)
 6341                ahd_chip_init(ahd);
 6342
 6343        return (0);
 6344}
 6345
 6346/*
 6347 * Determine the number of SCBs available on the controller
 6348 */
 6349static int
 6350ahd_probe_scbs(struct ahd_softc *ahd) {
 6351        int i;
 6352
 6353        AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
 6354                         ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
 6355        for (i = 0; i < AHD_SCB_MAX; i++) {
 6356                int j;
 6357
 6358                ahd_set_scbptr(ahd, i);
 6359                ahd_outw(ahd, SCB_BASE, i);
 6360                for (j = 2; j < 64; j++)
 6361                        ahd_outb(ahd, SCB_BASE+j, 0);
 6362                /* Start out life as unallocated (needing an abort) */
 6363                ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
 6364                if (ahd_inw_scbram(ahd, SCB_BASE) != i)
 6365                        break;
 6366                ahd_set_scbptr(ahd, 0);
 6367                if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
 6368                        break;
 6369        }
 6370        return (i);
 6371}
 6372
 6373static void
 6374ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 
 6375{
 6376        dma_addr_t *baddr;
 6377
 6378        baddr = (dma_addr_t *)arg;
 6379        *baddr = segs->ds_addr;
 6380}
 6381
 6382static void
 6383ahd_initialize_hscbs(struct ahd_softc *ahd)
 6384{
 6385        int i;
 6386
 6387        for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
 6388                ahd_set_scbptr(ahd, i);
 6389
 6390                /* Clear the control byte. */
 6391                ahd_outb(ahd, SCB_CONTROL, 0);
 6392
 6393                /* Set the next pointer */
 6394                ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
 6395        }
 6396}
 6397
 6398static int
 6399ahd_init_scbdata(struct ahd_softc *ahd)
 6400{
 6401        struct        scb_data *scb_data;
 6402        int        i;
 6403
 6404        scb_data = &ahd->scb_data;
 6405        TAILQ_INIT(&scb_data->free_scbs);
 6406        for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
 6407                LIST_INIT(&scb_data->free_scb_lists[i]);
 6408        LIST_INIT(&scb_data->any_dev_free_scb_list);
 6409        SLIST_INIT(&scb_data->hscb_maps);
 6410        SLIST_INIT(&scb_data->sg_maps);
 6411        SLIST_INIT(&scb_data->sense_maps);
 6412
 6413        /* Determine the number of hardware SCBs and initialize them */
 6414        scb_data->maxhscbs = ahd_probe_scbs(ahd);
 6415        if (scb_data->maxhscbs == 0) {
 6416                printf("%s: No SCB space found\n", ahd_name(ahd));
 6417                return (ENXIO);
 6418        }
 6419
 6420        ahd_initialize_hscbs(ahd);
 6421
 6422        /*
 6423         * Create our DMA tags.  These tags define the kinds of device
 6424         * accessible memory allocations and memory mappings we will
 6425         * need to perform during normal operation.
 6426         *
 6427         * Unless we need to further restrict the allocation, we rely
 6428         * on the restrictions of the parent dmat, hence the common
 6429         * use of MAXADDR and MAXSIZE.
 6430         */
 6431
 6432        /* DMA tag for our hardware scb structures */
 6433        if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
 6434                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
 6435                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
 6436                               /*highaddr*/BUS_SPACE_MAXADDR,
 6437                               /*filter*/NULL, /*filterarg*/NULL,
 6438                               PAGE_SIZE, /*nsegments*/1,
 6439                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
 6440                               /*flags*/0, &scb_data->hscb_dmat) != 0) {
 6441                goto error_exit;
 6442        }
 6443
 6444        scb_data->init_level++;
 6445
 6446        /* DMA tag for our S/G structures. */
 6447        if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
 6448                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
 6449                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
 6450                               /*highaddr*/BUS_SPACE_MAXADDR,
 6451                               /*filter*/NULL, /*filterarg*/NULL,
 6452                               ahd_sglist_allocsize(ahd), /*nsegments*/1,
 6453                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
 6454                               /*flags*/0, &scb_data->sg_dmat) != 0) {
 6455                goto error_exit;
 6456        }
 6457#ifdef AHD_DEBUG
 6458        if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
 6459                printf("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
 6460                       ahd_sglist_allocsize(ahd));
 6461#endif
 6462
 6463        scb_data->init_level++;
 6464
 6465        /* DMA tag for our sense buffers.  We allocate in page sized chunks */
 6466        if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
 6467                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
 6468                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
 6469                               /*highaddr*/BUS_SPACE_MAXADDR,
 6470                               /*filter*/NULL, /*filterarg*/NULL,
 6471                               PAGE_SIZE, /*nsegments*/1,
 6472                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
 6473                               /*flags*/0, &scb_data->sense_dmat) != 0) {
 6474                goto error_exit;
 6475        }
 6476
 6477        scb_data->init_level++;
 6478
 6479        /* Perform initial CCB allocation */
 6480        ahd_alloc_scbs(ahd);
 6481
 6482        if (scb_data->numscbs == 0) {
 6483                printf("%s: ahd_init_scbdata - "
 6484                       "Unable to allocate initial scbs\n",
 6485                       ahd_name(ahd));
 6486                goto error_exit;
 6487        }
 6488
 6489        /*
 6490         * Note that we were successfull
 6491         */
 6492        return (0); 
 6493
 6494error_exit:
 6495
 6496        return (ENOMEM);
 6497}
 6498
 6499static struct scb *
 6500ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
 6501{
 6502        struct scb *scb;
 6503
 6504        /*
 6505         * Look on the pending list.
 6506         */
 6507        LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
 6508                if (SCB_GET_TAG(scb) == tag)
 6509                        return (scb);
 6510        }
 6511
 6512        /*
 6513         * Then on all of the collision free lists.
 6514         */
 6515        TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
 6516                struct scb *list_scb;
 6517
 6518                list_scb = scb;
 6519                do {
 6520                        if (SCB_GET_TAG(list_scb) == tag)
 6521                                return (list_scb);
 6522                        list_scb = LIST_NEXT(list_scb, collision_links);
 6523                } while (list_scb);
 6524        }
 6525
 6526        /*
 6527         * And finally on the generic free list.
 6528         */
 6529        LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
 6530                if (SCB_GET_TAG(scb) == tag)
 6531                        return (scb);
 6532        }
 6533
 6534        return (NULL);
 6535}
 6536
 6537static void
 6538ahd_fini_scbdata(struct ahd_softc *ahd)
 6539{
 6540        struct scb_data *scb_data;
 6541
 6542        scb_data = &ahd->scb_data;
 6543        if (scb_data == NULL)
 6544                return;
 6545
 6546        switch (scb_data->init_level) {
 6547        default:
 6548        case 7:
 6549        {
 6550                struct map_node *sns_map;
 6551
 6552                while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
 6553                        SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
 6554                        ahd_dmamap_unload(ahd, scb_data->sense_dmat,
 6555                                          sns_map->dmamap);
 6556                        ahd_dmamem_free(ahd, scb_data->sense_dmat,
 6557                                        sns_map->vaddr, sns_map->dmamap);
 6558                        free(sns_map, M_DEVBUF);
 6559                }
 6560                ahd_dma_tag_destroy(ahd, scb_data->sense_dmat);
 6561                /* FALLTHROUGH */
 6562        }
 6563        case 6:
 6564        {
 6565                struct map_node *sg_map;
 6566
 6567                while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
 6568                        SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
 6569                        ahd_dmamap_unload(ahd, scb_data->sg_dmat,
 6570                                          sg_map->dmamap);
 6571                        ahd_dmamem_free(ahd, scb_data->sg_dmat,
 6572                                        sg_map->vaddr, sg_map->dmamap);
 6573                        free(sg_map, M_DEVBUF);
 6574                }
 6575                ahd_dma_tag_destroy(ahd, scb_data->sg_dmat);
 6576                /* FALLTHROUGH */
 6577        }
 6578        case 5:
 6579        {
 6580                struct map_node *hscb_map;
 6581
 6582                while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
 6583                        SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
 6584                        ahd_dmamap_unload(ahd, scb_data->hscb_dmat,
 6585                                          hscb_map->dmamap);
 6586                        ahd_dmamem_free(ahd, scb_data->hscb_dmat,
 6587                                        hscb_map->vaddr, hscb_map->dmamap);
 6588                        free(hscb_map, M_DEVBUF);
 6589                }
 6590                ahd_dma_tag_destroy(ahd, scb_data->hscb_dmat);
 6591                /* FALLTHROUGH */
 6592        }
 6593        case 4:
 6594        case 3:
 6595        case 2:
 6596        case 1:
 6597        case 0:
 6598                break;
 6599        }
 6600}
 6601
 6602/*
 6603 * DSP filter Bypass must be enabled until the first selection
 6604 * after a change in bus mode (Razor #491 and #493).
 6605 */
 6606static void
 6607ahd_setup_iocell_workaround(struct ahd_softc *ahd)
 6608{
 6609        ahd_mode_state saved_modes;
 6610
 6611        saved_modes = ahd_save_modes(ahd);
 6612        ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 6613        ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
 6614               | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
 6615        ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
 6616#ifdef AHD_DEBUG
 6617        if ((ahd_debug & AHD_SHOW_MISC) != 0)
 6618                printf("%s: Setting up iocell workaround\n", ahd_name(ahd));
 6619#endif
 6620        ahd_restore_modes(ahd, saved_modes);
 6621        ahd->flags &= ~AHD_HAD_FIRST_SEL;
 6622}
 6623
 6624static void
 6625ahd_iocell_first_selection(struct ahd_softc *ahd)
 6626{
 6627        ahd_mode_state        saved_modes;
 6628        u_int                sblkctl;
 6629
 6630        if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
 6631                return;
 6632        saved_modes = ahd_save_modes(ahd);
 6633        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 6634        sblkctl = ahd_inb(ahd, SBLKCTL);
 6635        ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 6636#ifdef AHD_DEBUG
 6637        if ((ahd_debug & AHD_SHOW_MISC) != 0)
 6638                printf("%s: iocell first selection\n", ahd_name(ahd));
 6639#endif
 6640        if ((sblkctl & ENAB40) != 0) {
 6641                ahd_outb(ahd, DSPDATACTL,
 6642                         ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
 6643#ifdef AHD_DEBUG
 6644                if ((ahd_debug & AHD_SHOW_MISC) != 0)
 6645                        printf("%s: BYPASS now disabled\n", ahd_name(ahd));
 6646#endif
 6647        }
 6648        ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
 6649        ahd_outb(ahd, CLRINT, CLRSCSIINT);
 6650        ahd_restore_modes(ahd, saved_modes);
 6651        ahd->flags |= AHD_HAD_FIRST_SEL;
 6652}
 6653
 6654/*************************** SCB Management ***********************************/
 6655static void
 6656ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
 6657{
 6658        struct        scb_list *free_list;
 6659        struct        scb_tailq *free_tailq;
 6660        struct        scb *first_scb;
 6661
 6662        scb->flags |= SCB_ON_COL_LIST;
 6663        AHD_SET_SCB_COL_IDX(scb, col_idx);
 6664        free_list = &ahd->scb_data.free_scb_lists[col_idx];
 6665        free_tailq = &ahd->scb_data.free_scbs;
 6666        first_scb = LIST_FIRST(free_list);
 6667        if (first_scb != NULL) {
 6668                LIST_INSERT_AFTER(first_scb, scb, collision_links);
 6669        } else {
 6670                LIST_INSERT_HEAD(free_list, scb, collision_links);
 6671                TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
 6672        }
 6673}
 6674
 6675static void
 6676ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
 6677{
 6678        struct        scb_list *free_list;
 6679        struct        scb_tailq *free_tailq;
 6680        struct        scb *first_scb;
 6681        u_int        col_idx;
 6682
 6683        scb->flags &= ~SCB_ON_COL_LIST;
 6684        col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
 6685        free_list = &ahd->scb_data.free_scb_lists[col_idx];
 6686        free_tailq = &ahd->scb_data.free_scbs;
 6687        first_scb = LIST_FIRST(free_list);
 6688        if (first_scb == scb) {
 6689                struct scb *next_scb;
 6690
 6691                /*
 6692                 * Maintain order in the collision free
 6693                 * lists for fairness if this device has
 6694                 * other colliding tags active.
 6695                 */
 6696                next_scb = LIST_NEXT(scb, collision_links);
 6697                if (next_scb != NULL) {
 6698                        TAILQ_INSERT_AFTER(free_tailq, scb,
 6699                                           next_scb, links.tqe);
 6700                }
 6701                TAILQ_REMOVE(free_tailq, scb, links.tqe);
 6702        }
 6703        LIST_REMOVE(scb, collision_links);
 6704}
 6705
 6706/*
 6707 * Get a free scb. If there are none, see if we can allocate a new SCB.
 6708 */
 6709struct scb *
 6710ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
 6711{
 6712        struct scb *scb;
 6713        int tries;
 6714
 6715        tries = 0;
 6716look_again:
 6717        TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
 6718                if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
 6719                        ahd_rem_col_list(ahd, scb);
 6720                        goto found;
 6721                }
 6722        }
 6723        if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
 6724
 6725                if (tries++ != 0)
 6726                        return (NULL);
 6727                ahd_alloc_scbs(ahd);
 6728                goto look_again;
 6729        }
 6730        LIST_REMOVE(scb, links.le);
 6731        if (col_idx != AHD_NEVER_COL_IDX
 6732         && (scb->col_scb != NULL)
 6733         && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
 6734                LIST_REMOVE(scb->col_scb, links.le);
 6735                ahd_add_col_list(ahd, scb->col_scb, col_idx);
 6736        }
 6737found:
 6738        scb->flags |= SCB_ACTIVE;
 6739        return (scb);
 6740}
 6741
 6742/*
 6743 * Return an SCB resource to the free list.
 6744 */
 6745void
 6746ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
 6747{
 6748        /* Clean up for the next user */
 6749        scb->flags = SCB_FLAG_NONE;
 6750        scb->hscb->control = 0;
 6751        ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
 6752
 6753        if (scb->col_scb == NULL) {
 6754
 6755                /*
 6756                 * No collision possible.  Just free normally.
 6757                 */
 6758                LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
 6759                                 scb, links.le);
 6760        } else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
 6761
 6762                /*
 6763                 * The SCB we might have collided with is on
 6764                 * a free collision list.  Put both SCBs on
 6765                 * the generic list.
 6766                 */
 6767                ahd_rem_col_list(ahd, scb->col_scb);
 6768                LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
 6769                                 scb, links.le);
 6770                LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
 6771                                 scb->col_scb, links.le);
 6772        } else if ((scb->col_scb->flags
 6773                  & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
 6774                && (scb->col_scb->hscb->control & TAG_ENB) != 0) {
 6775
 6776                /*
 6777                 * The SCB we might collide with on the next allocation
 6778                 * is still active in a non-packetized, tagged, context.
 6779                 * Put us on the SCB collision list.
 6780                 */
 6781                ahd_add_col_list(ahd, scb,
 6782                                 AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
 6783        } else {
 6784                /*
 6785                 * The SCB we might collide with on the next allocation
 6786                 * is either active in a packetized context, or free.
 6787                 * Since we can't collide, put this SCB on the generic
 6788                 * free list.
 6789                 */
 6790                LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
 6791                                 scb, links.le);
 6792        }
 6793
 6794        ahd_platform_scb_free(ahd, scb);
 6795}
 6796
 6797static void
 6798ahd_alloc_scbs(struct ahd_softc *ahd)
 6799{
 6800        struct scb_data *scb_data;
 6801        struct scb        *next_scb;
 6802        struct hardware_scb *hscb;
 6803        struct map_node *hscb_map;
 6804        struct map_node *sg_map;
 6805        struct map_node *sense_map;
 6806        uint8_t                *segs;
 6807        uint8_t                *sense_data;
 6808        dma_addr_t         hscb_busaddr;
 6809        dma_addr_t         sg_busaddr;
 6810        dma_addr_t         sense_busaddr;
 6811        int                 newcount;
 6812        int                 i;
 6813
 6814        scb_data = &ahd->scb_data;
 6815        if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
 6816                /* Can't allocate any more */
 6817                return;
 6818
 6819        if (scb_data->scbs_left != 0) {
 6820                int offset;
 6821
 6822                offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
 6823                hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
 6824                hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
 6825                hscb_busaddr = hscb_map->physaddr + (offset * sizeof(*hscb));
 6826        } else {
 6827                hscb_map = malloc(sizeof(*hscb_map), M_DEVBUF, M_NOWAIT);
 6828
 6829                if (hscb_map == NULL)
 6830                        return;
 6831
 6832                /* Allocate the next batch of hardware SCBs */
 6833                if (ahd_dmamem_alloc(ahd, scb_data->hscb_dmat,
 6834                                     (void **)&hscb_map->vaddr,
 6835                                     BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
 6836                        free(hscb_map, M_DEVBUF);
 6837                        return;
 6838                }
 6839
 6840                SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
 6841
 6842                ahd_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
 6843                                hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
 6844                                &hscb_map->physaddr, /*flags*/0);
 6845
 6846                hscb = (struct hardware_scb *)hscb_map->vaddr;
 6847                hscb_busaddr = hscb_map->physaddr;
 6848                scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
 6849        }
 6850
 6851        if (scb_data->sgs_left != 0) {
 6852                int offset;
 6853
 6854                offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
 6855                       - scb_data->sgs_left) * ahd_sglist_size(ahd);
 6856                sg_map = SLIST_FIRST(&scb_data->sg_maps);
 6857                segs = sg_map->vaddr + offset;
 6858                sg_busaddr = sg_map->physaddr + offset;
 6859        } else {
 6860                sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
 6861
 6862                if (sg_map == NULL)
 6863                        return;
 6864
 6865                /* Allocate the next batch of S/G lists */
 6866                if (ahd_dmamem_alloc(ahd, scb_data->sg_dmat,
 6867                                     (void **)&sg_map->vaddr,
 6868                                     BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
 6869                        free(sg_map, M_DEVBUF);
 6870                        return;
 6871                }
 6872
 6873                SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
 6874
 6875                ahd_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
 6876                                sg_map->vaddr, ahd_sglist_allocsize(ahd),
 6877                                ahd_dmamap_cb, &sg_map->physaddr, /*flags*/0);
 6878
 6879                segs = sg_map->vaddr;
 6880                sg_busaddr = sg_map->physaddr;
 6881                scb_data->sgs_left =
 6882                    ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
 6883#ifdef AHD_DEBUG
 6884                if (ahd_debug & AHD_SHOW_MEMORY)
 6885                        printf("Mapped SG data\n");
 6886#endif
 6887        }
 6888
 6889        if (scb_data->sense_left != 0) {
 6890                int offset;
 6891
 6892                offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
 6893                sense_map = SLIST_FIRST(&scb_data->sense_maps);
 6894                sense_data = sense_map->vaddr + offset;
 6895                sense_busaddr = sense_map->physaddr + offset;
 6896        } else {
 6897                sense_map = malloc(sizeof(*sense_map), M_DEVBUF, M_NOWAIT);
 6898
 6899                if (sense_map == NULL)
 6900                        return;
 6901
 6902                /* Allocate the next batch of sense buffers */
 6903                if (ahd_dmamem_alloc(ahd, scb_data->sense_dmat,
 6904                                     (void **)&sense_map->vaddr,
 6905                                     BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
 6906                        free(sense_map, M_DEVBUF);
 6907                        return;
 6908                }
 6909
 6910                SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
 6911
 6912                ahd_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
 6913                                sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
 6914                                &sense_map->physaddr, /*flags*/0);
 6915
 6916                sense_data = sense_map->vaddr;
 6917                sense_busaddr = sense_map->physaddr;
 6918                scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
 6919#ifdef AHD_DEBUG
 6920                if (ahd_debug & AHD_SHOW_MEMORY)
 6921                        printf("Mapped sense data\n");
 6922#endif
 6923        }
 6924
 6925        newcount = min(scb_data->sense_left, scb_data->scbs_left);
 6926        newcount = min(newcount, scb_data->sgs_left);
 6927        newcount = min(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
 6928        for (i = 0; i < newcount; i++) {
 6929                struct scb_platform_data *pdata;
 6930                u_int col_tag;
 6931#ifndef __linux__
 6932                int error;
 6933#endif
 6934
 6935                next_scb = (struct scb *)malloc(sizeof(*next_scb),
 6936                                                M_DEVBUF, M_NOWAIT);
 6937                if (next_scb == NULL)
 6938                        break;
 6939
 6940                pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
 6941                                                           M_DEVBUF, M_NOWAIT);
 6942                if (pdata == NULL) {
 6943                        free(next_scb, M_DEVBUF);
 6944                        break;
 6945                }
 6946                next_scb->platform_data = pdata;
 6947                next_scb->hscb_map = hscb_map;
 6948                next_scb->sg_map = sg_map;
 6949                next_scb->sense_map = sense_map;
 6950                next_scb->sg_list = segs;
 6951                next_scb->sense_data = sense_data;
 6952                next_scb->sense_busaddr = sense_busaddr;
 6953                memset(hscb, 0, sizeof(*hscb));
 6954                next_scb->hscb = hscb;
 6955                hscb->hscb_busaddr = ahd_htole32(hscb_busaddr);
 6956
 6957                /*
 6958                 * The sequencer always starts with the second entry.
 6959                 * The first entry is embedded in the scb.
 6960                 */
 6961                next_scb->sg_list_busaddr = sg_busaddr;
 6962                if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
 6963                        next_scb->sg_list_busaddr
 6964                            += sizeof(struct ahd_dma64_seg);
 6965                else
 6966                        next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
 6967                next_scb->ahd_softc = ahd;
 6968                next_scb->flags = SCB_FLAG_NONE;
 6969#ifndef __linux__
 6970                error = ahd_dmamap_create(ahd, ahd->buffer_dmat, /*flags*/0,
 6971                                          &next_scb->dmamap);
 6972                if (error != 0) {
 6973                        free(next_scb, M_DEVBUF);
 6974                        free(pdata, M_DEVBUF);
 6975                        break;
 6976                }
 6977#endif
 6978                next_scb->hscb->tag = ahd_htole16(scb_data->numscbs);
 6979                col_tag = scb_data->numscbs ^ 0x100;
 6980                next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
 6981                if (next_scb->col_scb != NULL)
 6982                        next_scb->col_scb->col_scb = next_scb;
 6983                ahd_free_scb(ahd, next_scb);
 6984                hscb++;
 6985                hscb_busaddr += sizeof(*hscb);
 6986                segs += ahd_sglist_size(ahd);
 6987                sg_busaddr += ahd_sglist_size(ahd);
 6988                sense_data += AHD_SENSE_BUFSIZE;
 6989                sense_busaddr += AHD_SENSE_BUFSIZE;
 6990                scb_data->numscbs++;
 6991                scb_data->sense_left--;
 6992                scb_data->scbs_left--;
 6993                scb_data->sgs_left--;
 6994        }
 6995}
 6996
 6997void
 6998ahd_controller_info(struct ahd_softc *ahd, char *buf)
 6999{
 7000        const char *speed;
 7001        const char *type;
 7002        int len;
 7003
 7004        len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
 7005        buf += len;
 7006
 7007        speed = "Ultra320 ";
 7008        if ((ahd->features & AHD_WIDE) != 0) {
 7009                type = "Wide ";
 7010        } else {
 7011                type = "Single ";
 7012        }
 7013        len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
 7014                      speed, type, ahd->channel, ahd->our_id);
 7015        buf += len;
 7016
 7017        sprintf(buf, "%s, %d SCBs", ahd->bus_description,
 7018                ahd->scb_data.maxhscbs);
 7019}
 7020
 7021static const char *channel_strings[] = {
 7022        "Primary Low",
 7023        "Primary High",
 7024        "Secondary Low", 
 7025        "Secondary High"
 7026};
 7027
 7028static const char *termstat_strings[] = {
 7029        "Terminated Correctly",
 7030        "Over Terminated",
 7031        "Under Terminated",
 7032        "Not Configured"
 7033};
 7034
 7035/***************************** Timer Facilities *******************************/
 7036#define ahd_timer_init init_timer
 7037#define ahd_timer_stop del_timer_sync
 7038typedef void ahd_linux_callback_t (u_long);
 7039
 7040static void
 7041ahd_timer_reset(ahd_timer_t *timer, int usec, ahd_callback_t *func, void *arg)
 7042{
 7043        struct ahd_softc *ahd;
 7044
 7045        ahd = (struct ahd_softc *)arg;
 7046        del_timer(timer);
 7047        timer->data = (u_long)arg;
 7048        timer->expires = jiffies + (usec * HZ)/1000000;
 7049        timer->function = (ahd_linux_callback_t*)func;
 7050        add_timer(timer);
 7051}
 7052
 7053/*
 7054 * Start the board, ready for normal operation
 7055 */
 7056int
 7057ahd_init(struct ahd_softc *ahd)
 7058{
 7059        uint8_t                *next_vaddr;
 7060        dma_addr_t         next_baddr;
 7061        size_t                 driver_data_size;
 7062        int                 i;
 7063        int                 error;
 7064        u_int                 warn_user;
 7065        uint8_t                 current_sensing;
 7066        uint8_t                 fstat;
 7067
 7068        AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 7069
 7070        ahd->stack_size = ahd_probe_stack_size(ahd);
 7071        ahd->saved_stack = malloc(ahd->stack_size * sizeof(uint16_t),
 7072                                  M_DEVBUF, M_NOWAIT);
 7073        if (ahd->saved_stack == NULL)
 7074                return (ENOMEM);
 7075
 7076        /*
 7077         * Verify that the compiler hasn't over-agressively
 7078         * padded important structures.
 7079         */
 7080        if (sizeof(struct hardware_scb) != 64)
 7081                panic("Hardware SCB size is incorrect");
 7082
 7083#ifdef AHD_DEBUG
 7084        if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
 7085                ahd->flags |= AHD_SEQUENCER_DEBUG;
 7086#endif
 7087
 7088        /*
 7089         * Default to allowing initiator operations.
 7090         */
 7091        ahd->flags |= AHD_INITIATORROLE;
 7092
 7093        /*
 7094         * Only allow target mode features if this unit has them enabled.
 7095         */
 7096        if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
 7097                ahd->features &= ~AHD_TARGETMODE;
 7098
 7099#ifndef __linux__
 7100        /* DMA tag for mapping buffers into device visible space. */
 7101        if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
 7102                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
 7103                               /*lowaddr*/ahd->flags & AHD_39BIT_ADDRESSING
 7104                                        ? (dma_addr_t)0x7FFFFFFFFFULL
 7105                                        : BUS_SPACE_MAXADDR_32BIT,
 7106                               /*highaddr*/BUS_SPACE_MAXADDR,
 7107                               /*filter*/NULL, /*filterarg*/NULL,
 7108                               /*maxsize*/(AHD_NSEG - 1) * PAGE_SIZE,
 7109                               /*nsegments*/AHD_NSEG,
 7110                               /*maxsegsz*/AHD_MAXTRANSFER_SIZE,
 7111                               /*flags*/BUS_DMA_ALLOCNOW,
 7112                               &ahd->buffer_dmat) != 0) {
 7113                return (ENOMEM);
 7114        }
 7115#endif
 7116
 7117        ahd->init_level++;
 7118
 7119        /*
 7120         * DMA tag for our command fifos and other data in system memory
 7121         * the card's sequencer must be able to access.  For initiator
 7122         * roles, we need to allocate space for the qoutfifo.  When providing
 7123         * for the target mode role, we must additionally provide space for
 7124         * the incoming target command fifo.
 7125         */
 7126        driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
 7127                         + sizeof(struct hardware_scb);
 7128        if ((ahd->features & AHD_TARGETMODE) != 0)
 7129                driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
 7130        if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
 7131                driver_data_size += PKT_OVERRUN_BUFSIZE;
 7132        if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
 7133                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
 7134                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
 7135                               /*highaddr*/BUS_SPACE_MAXADDR,
 7136                               /*filter*/NULL, /*filterarg*/NULL,
 7137                               driver_data_size,
 7138                               /*nsegments*/1,
 7139                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
 7140                               /*flags*/0, &ahd->shared_data_dmat) != 0) {
 7141                return (ENOMEM);
 7142        }
 7143
 7144        ahd->init_level++;
 7145
 7146        /* Allocation of driver data */
 7147        if (ahd_dmamem_alloc(ahd, ahd->shared_data_dmat,
 7148                             (void **)&ahd->shared_data_map.vaddr,
 7149                             BUS_DMA_NOWAIT,
 7150                             &ahd->shared_data_map.dmamap) != 0) {
 7151                return (ENOMEM);
 7152        }
 7153
 7154        ahd->init_level++;
 7155
 7156        /* And permanently map it in */
 7157        ahd_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
 7158                        ahd->shared_data_map.vaddr, driver_data_size,
 7159                        ahd_dmamap_cb, &ahd->shared_data_map.physaddr,
 7160                        /*flags*/0);
 7161        ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
 7162        next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
 7163        next_baddr = ahd->shared_data_map.physaddr
 7164                   + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
 7165        if ((ahd->features & AHD_TARGETMODE) != 0) {
 7166                ahd->targetcmds = (struct target_cmd *)next_vaddr;
 7167                next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
 7168                next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
 7169        }
 7170
 7171        if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
 7172                ahd->overrun_buf = next_vaddr;
 7173                next_vaddr += PKT_OVERRUN_BUFSIZE;
 7174                next_baddr += PKT_OVERRUN_BUFSIZE;
 7175        }
 7176
 7177        /*
 7178         * We need one SCB to serve as the "next SCB".  Since the
 7179         * tag identifier in this SCB will never be used, there is
 7180         * no point in using a valid HSCB tag from an SCB pulled from
 7181         * the standard free pool.  So, we allocate this "sentinel"
 7182         * specially from the DMA safe memory chunk used for the QOUTFIFO.
 7183         */
 7184        ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
 7185        ahd->next_queued_hscb_map = &ahd->shared_data_map;
 7186        ahd->next_queued_hscb->hscb_busaddr = ahd_htole32(next_baddr);
 7187
 7188        ahd->init_level++;
 7189
 7190        /* Allocate SCB data now that buffer_dmat is initialized */
 7191        if (ahd_init_scbdata(ahd) != 0)
 7192                return (ENOMEM);
 7193
 7194        if ((ahd->flags & AHD_INITIATORROLE) == 0)
 7195                ahd->flags &= ~AHD_RESET_BUS_A;
 7196
 7197        /*
 7198         * Before committing these settings to the chip, give
 7199         * the OSM one last chance to modify our configuration.
 7200         */
 7201        ahd_platform_init(ahd);
 7202
 7203        /* Bring up the chip. */
 7204        ahd_chip_init(ahd);
 7205
 7206        AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
 7207
 7208        if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
 7209                goto init_done;
 7210
 7211        /*
 7212         * Verify termination based on current draw and
 7213         * warn user if the bus is over/under terminated.
 7214         */
 7215        error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
 7216                                   CURSENSE_ENB);
 7217        if (error != 0) {
 7218                printf("%s: current sensing timeout 1\n", ahd_name(ahd));
 7219                goto init_done;
 7220        }
 7221        for (i = 20, fstat = FLX_FSTAT_BUSY;
 7222             (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
 7223                error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
 7224                if (error != 0) {
 7225                        printf("%s: current sensing timeout 2\n",
 7226                               ahd_name(ahd));
 7227                        goto init_done;
 7228                }
 7229        }
 7230        if (i == 0) {
 7231                printf("%s: Timedout during current-sensing test\n",
 7232                       ahd_name(ahd));
 7233                goto init_done;
 7234        }
 7235
 7236        /* Latch Current Sensing status. */
 7237        error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
 7238        if (error != 0) {
 7239                printf("%s: current sensing timeout 3\n", ahd_name(ahd));
 7240                goto init_done;
 7241        }
 7242
 7243        /* Diable current sensing. */
 7244        ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
 7245
 7246#ifdef AHD_DEBUG
 7247        if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
 7248                printf("%s: current_sensing == 0x%x\n",
 7249                       ahd_name(ahd), current_sensing);
 7250        }
 7251#endif
 7252        warn_user = 0;
 7253        for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
 7254                u_int term_stat;
 7255
 7256                term_stat = (current_sensing & FLX_CSTAT_MASK);
 7257                switch (term_stat) {
 7258                case FLX_CSTAT_OVER:
 7259                case FLX_CSTAT_UNDER:
 7260                        warn_user++;
 7261                case FLX_CSTAT_INVALID:
 7262                case FLX_CSTAT_OKAY:
 7263                        if (warn_user == 0 && bootverbose == 0)
 7264                                break;
 7265                        printf("%s: %s Channel %s\n", ahd_name(ahd),
 7266                               channel_strings[i], termstat_strings[term_stat]);
 7267                        break;
 7268                }
 7269        }
 7270        if (warn_user) {
 7271                printf("%s: WARNING. Termination is not configured correctly.\n"
 7272                       "%s: WARNING. SCSI bus operations may FAIL.\n",
 7273                       ahd_name(ahd), ahd_name(ahd));
 7274        }
 7275init_done:
 7276        ahd_restart(ahd);
 7277        ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US,
 7278                        ahd_stat_timer, ahd);
 7279        return (0);
 7280}
 7281
 7282/*
 7283 * (Re)initialize chip state after a chip reset.
 7284 */
 7285static void
 7286ahd_chip_init(struct ahd_softc *ahd)
 7287{
 7288        uint32_t busaddr;
 7289        u_int         sxfrctl1;
 7290        u_int         scsiseq_template;
 7291        u_int         wait;
 7292        u_int         i;
 7293        u_int         target;
 7294
 7295        ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
 7296        /*
 7297         * Take the LED out of diagnostic mode
 7298         */
 7299        ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
 7300
 7301        /*
 7302         * Return HS_MAILBOX to its default value.
 7303         */
 7304        ahd->hs_mailbox = 0;
 7305        ahd_outb(ahd, HS_MAILBOX, 0);
 7306
 7307        /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
 7308        ahd_outb(ahd, IOWNID, ahd->our_id);
 7309        ahd_outb(ahd, TOWNID, ahd->our_id);
 7310        sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
 7311        sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
 7312        if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
 7313         && (ahd->seltime != STIMESEL_MIN)) {
 7314                /*
 7315                 * The selection timer duration is twice as long
 7316                 * as it should be.  Halve it by adding "1" to
 7317                 * the user specified setting.
 7318                 */
 7319                sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
 7320        } else {
 7321                sxfrctl1 |= ahd->seltime;
 7322        }
 7323                
 7324        ahd_outb(ahd, SXFRCTL0, DFON);
 7325        ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
 7326        ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
 7327
 7328        /*
 7329         * Now that termination is set, wait for up
 7330         * to 500ms for our transceivers to settle.  If
 7331         * the adapter does not have a cable attached,
 7332         * the transceivers may never settle, so don't
 7333         * complain if we fail here.
 7334         */
 7335        for (wait = 10000;
 7336             (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
 7337             wait--)
 7338                ahd_delay(100);
 7339
 7340        /* Clear any false bus resets due to the transceivers settling */
 7341        ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
 7342        ahd_outb(ahd, CLRINT, CLRSCSIINT);
 7343
 7344        /* Initialize mode specific S/G state. */
 7345        for (i = 0; i < 2; i++) {
 7346                ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
 7347                ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
 7348                ahd_outb(ahd, SG_STATE, 0);
 7349                ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
 7350                ahd_outb(ahd, SEQIMODE,
 7351                         ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
 7352                        |ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
 7353        }
 7354
 7355        ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
 7356        ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
 7357        ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
 7358        ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
 7359        ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
 7360        if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
 7361                ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
 7362        } else {
 7363                ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
 7364        }
 7365        ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
 7366        if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
 7367                /*
 7368                 * Do not issue a target abort when a split completion
 7369                 * error occurs.  Let our PCIX interrupt handler deal
 7370                 * with it instead. H2A4 Razor #625
 7371                 */
 7372                ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
 7373
 7374        if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
 7375                ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
 7376
 7377        /*
 7378         * Tweak IOCELL settings.
 7379         */
 7380        if ((ahd->flags & AHD_HP_BOARD) != 0) {
 7381                for (i = 0; i < NUMDSPS; i++) {
 7382                        ahd_outb(ahd, DSPSELECT, i);
 7383                        ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
 7384                }
 7385#ifdef AHD_DEBUG
 7386                if ((ahd_debug & AHD_SHOW_MISC) != 0)
 7387                        printf("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
 7388                               WRTBIASCTL_HP_DEFAULT);
 7389#endif
 7390        }
 7391        ahd_setup_iocell_workaround(ahd);
 7392
 7393        /*
 7394         * Enable LQI Manager interrupts.
 7395         */
 7396        ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
 7397                              | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
 7398                              | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
 7399        ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
 7400        /*
 7401         * We choose to have the sequencer catch LQOPHCHGINPKT errors
 7402         * manually for the command phase at the start of a packetized
 7403         * selection case.  ENLQOBUSFREE should be made redundant by
 7404         * the BUSFREE interrupt, but it seems that some LQOBUSFREE
 7405         * events fail to assert the BUSFREE interrupt so we must
 7406         * also enable LQOBUSFREE interrupts.
 7407         */
 7408        ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
 7409
 7410        /*
 7411         * Setup sequencer interrupt handlers.
 7412         */
 7413        ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
 7414        ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
 7415
 7416        /*
 7417         * Setup SCB Offset registers.
 7418         */
 7419        if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
 7420                ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
 7421                         pkt_long_lun));
 7422        } else {
 7423                ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
 7424        }
 7425        ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
 7426        ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
 7427        ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
 7428        ahd_outb(ahd, CMDPTR, offsetof(struct