Showing error 1837

User: Jiri Slaby
Error type: Invalid Pointer Dereference
Error type description: A pointer which is invalid is being dereferenced
File location: drivers/scsi/aic7xxx/aic7xxx_core.c
Line in file: 7141
Project: Linux Kernel
Project version: 2.6.28
Tools: Smatch (1.59)
Entered: 2013-09-11 08:47:26 UTC


Source:

   1/*
   2 * Core routines and tables shareable across OS platforms.
   3 *
   4 * Copyright (c) 1994-2002 Justin T. Gibbs.
   5 * Copyright (c) 2000-2002 Adaptec Inc.
   6 * All rights reserved.
   7 *
   8 * Redistribution and use in source and binary forms, with or without
   9 * modification, are permitted provided that the following conditions
  10 * are met:
  11 * 1. Redistributions of source code must retain the above copyright
  12 *    notice, this list of conditions, and the following disclaimer,
  13 *    without modification.
  14 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15 *    substantially similar to the "NO WARRANTY" disclaimer below
  16 *    ("Disclaimer") and any redistribution must be conditioned upon
  17 *    including a substantially similar Disclaimer requirement for further
  18 *    binary redistribution.
  19 * 3. Neither the names of the above-listed copyright holders nor the names
  20 *    of any contributors may be used to endorse or promote products derived
  21 *    from this software without specific prior written permission.
  22 *
  23 * Alternatively, this software may be distributed under the terms of the
  24 * GNU General Public License ("GPL") version 2 as published by the Free
  25 * Software Foundation.
  26 *
  27 * NO WARRANTY
  28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38 * POSSIBILITY OF SUCH DAMAGES.
  39 *
  40 * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
  41 */
  42
  43#ifdef __linux__
  44#include "aic7xxx_osm.h"
  45#include "aic7xxx_inline.h"
  46#include "aicasm/aicasm_insformat.h"
  47#else
  48#include <dev/aic7xxx/aic7xxx_osm.h>
  49#include <dev/aic7xxx/aic7xxx_inline.h>
  50#include <dev/aic7xxx/aicasm/aicasm_insformat.h>
  51#endif
  52
  53/***************************** Lookup Tables **********************************/
  54static const char *const ahc_chip_names[] = {
  55        "NONE",
  56        "aic7770",
  57        "aic7850",
  58        "aic7855",
  59        "aic7859",
  60        "aic7860",
  61        "aic7870",
  62        "aic7880",
  63        "aic7895",
  64        "aic7895C",
  65        "aic7890/91",
  66        "aic7896/97",
  67        "aic7892",
  68        "aic7899"
  69};
  70static const u_int num_chip_names = ARRAY_SIZE(ahc_chip_names);
  71
  72/*
  73 * Hardware error codes.
  74 */
  75struct ahc_hard_error_entry {
  76        uint8_t errno;
  77        const char *errmesg;
  78};
  79
  80static const struct ahc_hard_error_entry ahc_hard_errors[] = {
  81        { ILLHADDR,        "Illegal Host Access" },
  82        { ILLSADDR,        "Illegal Sequencer Address referrenced" },
  83        { ILLOPCODE,        "Illegal Opcode in sequencer program" },
  84        { SQPARERR,        "Sequencer Parity Error" },
  85        { DPARERR,        "Data-path Parity Error" },
  86        { MPARERR,        "Scratch or SCB Memory Parity Error" },
  87        { PCIERRSTAT,        "PCI Error detected" },
  88        { CIOPARERR,        "CIOBUS Parity Error" },
  89};
  90static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);
  91
  92static const struct ahc_phase_table_entry ahc_phase_table[] =
  93{
  94        { P_DATAOUT,        MSG_NOOP,                "in Data-out phase"        },
  95        { P_DATAIN,        MSG_INITIATOR_DET_ERR,        "in Data-in phase"        },
  96        { P_DATAOUT_DT,        MSG_NOOP,                "in DT Data-out phase"        },
  97        { P_DATAIN_DT,        MSG_INITIATOR_DET_ERR,        "in DT Data-in phase"        },
  98        { P_COMMAND,        MSG_NOOP,                "in Command phase"        },
  99        { P_MESGOUT,        MSG_NOOP,                "in Message-out phase"        },
 100        { P_STATUS,        MSG_INITIATOR_DET_ERR,        "in Status phase"        },
 101        { P_MESGIN,        MSG_PARITY_ERROR,        "in Message-in phase"        },
 102        { P_BUSFREE,        MSG_NOOP,                "while idle"                },
 103        { 0,                MSG_NOOP,                "in unknown phase"        }
 104};
 105
 106/*
 107 * In most cases we only wish to itterate over real phases, so
 108 * exclude the last element from the count.
 109 */
 110static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;
 111
 112/*
 113 * Valid SCSIRATE values.  (p. 3-17)
 114 * Provides a mapping of tranfer periods in ns to the proper value to
 115 * stick in the scsixfer reg.
 116 */
 117static const struct ahc_syncrate ahc_syncrates[] =
 118{
 119      /* ultra2    fast/ultra  period     rate */
 120        { 0x42,      0x000,      9,      "80.0" },
 121        { 0x03,      0x000,     10,      "40.0" },
 122        { 0x04,      0x000,     11,      "33.0" },
 123        { 0x05,      0x100,     12,      "20.0" },
 124        { 0x06,      0x110,     15,      "16.0" },
 125        { 0x07,      0x120,     18,      "13.4" },
 126        { 0x08,      0x000,     25,      "10.0" },
 127        { 0x19,      0x010,     31,      "8.0"  },
 128        { 0x1a,      0x020,     37,      "6.67" },
 129        { 0x1b,      0x030,     43,      "5.7"  },
 130        { 0x1c,      0x040,     50,      "5.0"  },
 131        { 0x00,      0x050,     56,      "4.4"  },
 132        { 0x00,      0x060,     62,      "4.0"  },
 133        { 0x00,      0x070,     68,      "3.6"  },
 134        { 0x00,      0x000,      0,      NULL   }
 135};
 136
 137/* Our Sequencer Program */
 138#include "aic7xxx_seq.h"
 139
 140/**************************** Function Declarations ***************************/
 141static void                ahc_force_renegotiation(struct ahc_softc *ahc,
 142                                                struct ahc_devinfo *devinfo);
 143static struct ahc_tmode_tstate*
 144                        ahc_alloc_tstate(struct ahc_softc *ahc,
 145                                         u_int scsi_id, char channel);
 146#ifdef AHC_TARGET_MODE
 147static void                ahc_free_tstate(struct ahc_softc *ahc,
 148                                        u_int scsi_id, char channel, int force);
 149#endif
 150static const struct ahc_syncrate*
 151                        ahc_devlimited_syncrate(struct ahc_softc *ahc,
 152                                                struct ahc_initiator_tinfo *,
 153                                                u_int *period,
 154                                                u_int *ppr_options,
 155                                                role_t role);
 156static void                ahc_update_pending_scbs(struct ahc_softc *ahc);
 157static void                ahc_fetch_devinfo(struct ahc_softc *ahc,
 158                                          struct ahc_devinfo *devinfo);
 159static void                ahc_scb_devinfo(struct ahc_softc *ahc,
 160                                        struct ahc_devinfo *devinfo,
 161                                        struct scb *scb);
 162static void                ahc_assert_atn(struct ahc_softc *ahc);
 163static void                ahc_setup_initiator_msgout(struct ahc_softc *ahc,
 164                                                   struct ahc_devinfo *devinfo,
 165                                                   struct scb *scb);
 166static void                ahc_build_transfer_msg(struct ahc_softc *ahc,
 167                                               struct ahc_devinfo *devinfo);
 168static void                ahc_construct_sdtr(struct ahc_softc *ahc,
 169                                           struct ahc_devinfo *devinfo,
 170                                           u_int period, u_int offset);
 171static void                ahc_construct_wdtr(struct ahc_softc *ahc,
 172                                           struct ahc_devinfo *devinfo,
 173                                           u_int bus_width);
 174static void                ahc_construct_ppr(struct ahc_softc *ahc,
 175                                          struct ahc_devinfo *devinfo,
 176                                          u_int period, u_int offset,
 177                                          u_int bus_width, u_int ppr_options);
 178static void                ahc_clear_msg_state(struct ahc_softc *ahc);
 179static void                ahc_handle_proto_violation(struct ahc_softc *ahc);
 180static void                ahc_handle_message_phase(struct ahc_softc *ahc);
 181typedef enum {
 182        AHCMSG_1B,
 183        AHCMSG_2B,
 184        AHCMSG_EXT
 185} ahc_msgtype;
 186static int                ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
 187                                     u_int msgval, int full);
 188static int                ahc_parse_msg(struct ahc_softc *ahc,
 189                                      struct ahc_devinfo *devinfo);
 190static int                ahc_handle_msg_reject(struct ahc_softc *ahc,
 191                                              struct ahc_devinfo *devinfo);
 192static void                ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
 193                                                struct ahc_devinfo *devinfo);
 194static void                ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
 195static void                ahc_handle_devreset(struct ahc_softc *ahc,
 196                                            struct ahc_devinfo *devinfo,
 197                                            cam_status status, char *message,
 198                                            int verbose_level);
 199#ifdef AHC_TARGET_MODE
 200static void                ahc_setup_target_msgin(struct ahc_softc *ahc,
 201                                               struct ahc_devinfo *devinfo,
 202                                               struct scb *scb);
 203#endif
 204
 205static bus_dmamap_callback_t        ahc_dmamap_cb; 
 206static void                ahc_build_free_scb_list(struct ahc_softc *ahc);
 207static int                ahc_init_scbdata(struct ahc_softc *ahc);
 208static void                ahc_fini_scbdata(struct ahc_softc *ahc);
 209static void                ahc_qinfifo_requeue(struct ahc_softc *ahc,
 210                                            struct scb *prev_scb,
 211                                            struct scb *scb);
 212static int                ahc_qinfifo_count(struct ahc_softc *ahc);
 213static u_int                ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
 214                                                   u_int prev, u_int scbptr);
 215static void                ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
 216static u_int                ahc_rem_wscb(struct ahc_softc *ahc,
 217                                     u_int scbpos, u_int prev);
 218static void                ahc_reset_current_bus(struct ahc_softc *ahc);
 219#ifdef AHC_DUMP_SEQ
 220static void                ahc_dumpseq(struct ahc_softc *ahc);
 221#endif
 222static int                ahc_loadseq(struct ahc_softc *ahc);
 223static int                ahc_check_patch(struct ahc_softc *ahc,
 224                                        const struct patch **start_patch,
 225                                        u_int start_instr, u_int *skip_addr);
 226static void                ahc_download_instr(struct ahc_softc *ahc,
 227                                           u_int instrptr, uint8_t *dconsts);
 228#ifdef AHC_TARGET_MODE
 229static void                ahc_queue_lstate_event(struct ahc_softc *ahc,
 230                                               struct ahc_tmode_lstate *lstate,
 231                                               u_int initiator_id,
 232                                               u_int event_type,
 233                                               u_int event_arg);
 234static void                ahc_update_scsiid(struct ahc_softc *ahc,
 235                                          u_int targid_mask);
 236static int                ahc_handle_target_cmd(struct ahc_softc *ahc,
 237                                              struct target_cmd *cmd);
 238#endif
 239
 240static u_int                ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl);
 241static void                ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl);
 242static void                ahc_busy_tcl(struct ahc_softc *ahc,
 243                                     u_int tcl, u_int busyid);
 244
 245/************************** SCB and SCB queue management **********************/
 246static void                ahc_run_untagged_queues(struct ahc_softc *ahc);
 247static void                ahc_run_untagged_queue(struct ahc_softc *ahc,
 248                                               struct scb_tailq *queue);
 249
 250/****************************** Initialization ********************************/
 251static void                 ahc_alloc_scbs(struct ahc_softc *ahc);
 252static void                 ahc_shutdown(void *arg);
 253
 254/*************************** Interrupt Services *******************************/
 255static void                ahc_clear_intstat(struct ahc_softc *ahc);
 256static void                ahc_run_qoutfifo(struct ahc_softc *ahc);
 257#ifdef AHC_TARGET_MODE
 258static void                ahc_run_tqinfifo(struct ahc_softc *ahc, int paused);
 259#endif
 260static void                ahc_handle_brkadrint(struct ahc_softc *ahc);
 261static void                ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat);
 262static void                ahc_handle_scsiint(struct ahc_softc *ahc,
 263                                           u_int intstat);
 264static void                ahc_clear_critical_section(struct ahc_softc *ahc);
 265
 266/***************************** Error Recovery *********************************/
 267static void                ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb);
 268static int                ahc_abort_scbs(struct ahc_softc *ahc, int target,
 269                                       char channel, int lun, u_int tag,
 270                                       role_t role, uint32_t status);
 271static void                ahc_calc_residual(struct ahc_softc *ahc,
 272                                          struct scb *scb);
 273
 274/*********************** Untagged Transaction Routines ************************/
 275static inline void        ahc_freeze_untagged_queues(struct ahc_softc *ahc);
 276static inline void        ahc_release_untagged_queues(struct ahc_softc *ahc);
 277
 278/*
 279 * Block our completion routine from starting the next untagged
 280 * transaction for this target or target lun.
 281 */
 282static inline void
 283ahc_freeze_untagged_queues(struct ahc_softc *ahc)
 284{
 285        if ((ahc->flags & AHC_SCB_BTT) == 0)
 286                ahc->untagged_queue_lock++;
 287}
 288
 289/*
 290 * Allow the next untagged transaction for this target or target lun
 291 * to be executed.  We use a counting semaphore to allow the lock
 292 * to be acquired recursively.  Once the count drops to zero, the
 293 * transaction queues will be run.
 294 */
 295static inline void
 296ahc_release_untagged_queues(struct ahc_softc *ahc)
 297{
 298        if ((ahc->flags & AHC_SCB_BTT) == 0) {
 299                ahc->untagged_queue_lock--;
 300                if (ahc->untagged_queue_lock == 0)
 301                        ahc_run_untagged_queues(ahc);
 302        }
 303}
 304
 305/************************* Sequencer Execution Control ************************/
 306/*
 307 * Work around any chip bugs related to halting sequencer execution.
 308 * On Ultra2 controllers, we must clear the CIOBUS stretch signal by
 309 * reading a register that will set this signal and deassert it.
 310 * Without this workaround, if the chip is paused, by an interrupt or
 311 * manual pause while accessing scb ram, accesses to certain registers
 312 * will hang the system (infinite pci retries).
 313 */
 314static void
 315ahc_pause_bug_fix(struct ahc_softc *ahc)
 316{
 317        if ((ahc->features & AHC_ULTRA2) != 0)
 318                (void)ahc_inb(ahc, CCSCBCTL);
 319}
 320
 321/*
 322 * Determine whether the sequencer has halted code execution.
 323 * Returns non-zero status if the sequencer is stopped.
 324 */
 325int
 326ahc_is_paused(struct ahc_softc *ahc)
 327{
 328        return ((ahc_inb(ahc, HCNTRL) & PAUSE) != 0);
 329}
 330
 331/*
 332 * Request that the sequencer stop and wait, indefinitely, for it
 333 * to stop.  The sequencer will only acknowledge that it is paused
 334 * once it has reached an instruction boundary and PAUSEDIS is
 335 * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 336 * for critical sections.
 337 */
 338void
 339ahc_pause(struct ahc_softc *ahc)
 340{
 341        ahc_outb(ahc, HCNTRL, ahc->pause);
 342
 343        /*
 344         * Since the sequencer can disable pausing in a critical section, we
 345         * must loop until it actually stops.
 346         */
 347        while (ahc_is_paused(ahc) == 0)
 348                ;
 349
 350        ahc_pause_bug_fix(ahc);
 351}
 352
 353/*
 354 * Allow the sequencer to continue program execution.
 355 * We check here to ensure that no additional interrupt
 356 * sources that would cause the sequencer to halt have been
 357 * asserted.  If, for example, a SCSI bus reset is detected
 358 * while we are fielding a different, pausing, interrupt type,
 359 * we don't want to release the sequencer before going back
 360 * into our interrupt handler and dealing with this new
 361 * condition.
 362 */
 363void
 364ahc_unpause(struct ahc_softc *ahc)
 365{
 366        if ((ahc_inb(ahc, INTSTAT) & (SCSIINT | SEQINT | BRKADRINT)) == 0)
 367                ahc_outb(ahc, HCNTRL, ahc->unpause);
 368}
 369
 370/************************** Memory mapping routines ***************************/
 371static struct ahc_dma_seg *
 372ahc_sg_bus_to_virt(struct scb *scb, uint32_t sg_busaddr)
 373{
 374        int sg_index;
 375
 376        sg_index = (sg_busaddr - scb->sg_list_phys)/sizeof(struct ahc_dma_seg);
 377        /* sg_list_phys points to entry 1, not 0 */
 378        sg_index++;
 379
 380        return (&scb->sg_list[sg_index]);
 381}
 382
 383static uint32_t
 384ahc_sg_virt_to_bus(struct scb *scb, struct ahc_dma_seg *sg)
 385{
 386        int sg_index;
 387
 388        /* sg_list_phys points to entry 1, not 0 */
 389        sg_index = sg - &scb->sg_list[1];
 390
 391        return (scb->sg_list_phys + (sg_index * sizeof(*scb->sg_list)));
 392}
 393
 394static uint32_t
 395ahc_hscb_busaddr(struct ahc_softc *ahc, u_int index)
 396{
 397        return (ahc->scb_data->hscb_busaddr
 398                + (sizeof(struct hardware_scb) * index));
 399}
 400
 401static void
 402ahc_sync_scb(struct ahc_softc *ahc, struct scb *scb, int op)
 403{
 404        ahc_dmamap_sync(ahc, ahc->scb_data->hscb_dmat,
 405                        ahc->scb_data->hscb_dmamap,
 406                        /*offset*/(scb->hscb - ahc->hscbs) * sizeof(*scb->hscb),
 407                        /*len*/sizeof(*scb->hscb), op);
 408}
 409
 410void
 411ahc_sync_sglist(struct ahc_softc *ahc, struct scb *scb, int op)
 412{
 413        if (scb->sg_count == 0)
 414                return;
 415
 416        ahc_dmamap_sync(ahc, ahc->scb_data->sg_dmat, scb->sg_map->sg_dmamap,
 417                        /*offset*/(scb->sg_list - scb->sg_map->sg_vaddr)
 418                                * sizeof(struct ahc_dma_seg),
 419                        /*len*/sizeof(struct ahc_dma_seg) * scb->sg_count, op);
 420}
 421
 422#ifdef AHC_TARGET_MODE
 423static uint32_t
 424ahc_targetcmd_offset(struct ahc_softc *ahc, u_int index)
 425{
 426        return (((uint8_t *)&ahc->targetcmds[index]) - ahc->qoutfifo);
 427}
 428#endif
 429
 430/*********************** Miscelaneous Support Functions ***********************/
 431/*
 432 * Determine whether the sequencer reported a residual
 433 * for this SCB/transaction.
 434 */
 435static void
 436ahc_update_residual(struct ahc_softc *ahc, struct scb *scb)
 437{
 438        uint32_t sgptr;
 439
 440        sgptr = ahc_le32toh(scb->hscb->sgptr);
 441        if ((sgptr & SG_RESID_VALID) != 0)
 442                ahc_calc_residual(ahc, scb);
 443}
 444
 445/*
 446 * Return pointers to the transfer negotiation information
 447 * for the specified our_id/remote_id pair.
 448 */
 449struct ahc_initiator_tinfo *
 450ahc_fetch_transinfo(struct ahc_softc *ahc, char channel, u_int our_id,
 451                    u_int remote_id, struct ahc_tmode_tstate **tstate)
 452{
 453        /*
 454         * Transfer data structures are stored from the perspective
 455         * of the target role.  Since the parameters for a connection
 456         * in the initiator role to a given target are the same as
 457         * when the roles are reversed, we pretend we are the target.
 458         */
 459        if (channel == 'B')
 460                our_id += 8;
 461        *tstate = ahc->enabled_targets[our_id];
 462        return (&(*tstate)->transinfo[remote_id]);
 463}
 464
 465uint16_t
 466ahc_inw(struct ahc_softc *ahc, u_int port)
 467{
 468        uint16_t r = ahc_inb(ahc, port+1) << 8;
 469        return r | ahc_inb(ahc, port);
 470}
 471
 472void
 473ahc_outw(struct ahc_softc *ahc, u_int port, u_int value)
 474{
 475        ahc_outb(ahc, port, value & 0xFF);
 476        ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 477}
 478
 479uint32_t
 480ahc_inl(struct ahc_softc *ahc, u_int port)
 481{
 482        return ((ahc_inb(ahc, port))
 483              | (ahc_inb(ahc, port+1) << 8)
 484              | (ahc_inb(ahc, port+2) << 16)
 485              | (ahc_inb(ahc, port+3) << 24));
 486}
 487
 488void
 489ahc_outl(struct ahc_softc *ahc, u_int port, uint32_t value)
 490{
 491        ahc_outb(ahc, port, (value) & 0xFF);
 492        ahc_outb(ahc, port+1, ((value) >> 8) & 0xFF);
 493        ahc_outb(ahc, port+2, ((value) >> 16) & 0xFF);
 494        ahc_outb(ahc, port+3, ((value) >> 24) & 0xFF);
 495}
 496
 497uint64_t
 498ahc_inq(struct ahc_softc *ahc, u_int port)
 499{
 500        return ((ahc_inb(ahc, port))
 501              | (ahc_inb(ahc, port+1) << 8)
 502              | (ahc_inb(ahc, port+2) << 16)
 503              | (ahc_inb(ahc, port+3) << 24)
 504              | (((uint64_t)ahc_inb(ahc, port+4)) << 32)
 505              | (((uint64_t)ahc_inb(ahc, port+5)) << 40)
 506              | (((uint64_t)ahc_inb(ahc, port+6)) << 48)
 507              | (((uint64_t)ahc_inb(ahc, port+7)) << 56));
 508}
 509
 510void
 511ahc_outq(struct ahc_softc *ahc, u_int port, uint64_t value)
 512{
 513        ahc_outb(ahc, port, value & 0xFF);
 514        ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 515        ahc_outb(ahc, port+2, (value >> 16) & 0xFF);
 516        ahc_outb(ahc, port+3, (value >> 24) & 0xFF);
 517        ahc_outb(ahc, port+4, (value >> 32) & 0xFF);
 518        ahc_outb(ahc, port+5, (value >> 40) & 0xFF);
 519        ahc_outb(ahc, port+6, (value >> 48) & 0xFF);
 520        ahc_outb(ahc, port+7, (value >> 56) & 0xFF);
 521}
 522
 523/*
 524 * Get a free scb. If there are none, see if we can allocate a new SCB.
 525 */
 526struct scb *
 527ahc_get_scb(struct ahc_softc *ahc)
 528{
 529        struct scb *scb;
 530
 531        if ((scb = SLIST_FIRST(&ahc->scb_data->free_scbs)) == NULL) {
 532                ahc_alloc_scbs(ahc);
 533                scb = SLIST_FIRST(&ahc->scb_data->free_scbs);
 534                if (scb == NULL)
 535                        return (NULL);
 536        }
 537        SLIST_REMOVE_HEAD(&ahc->scb_data->free_scbs, links.sle);
 538        return (scb);
 539}
 540
 541/*
 542 * Return an SCB resource to the free list.
 543 */
 544void
 545ahc_free_scb(struct ahc_softc *ahc, struct scb *scb)
 546{
 547        struct hardware_scb *hscb;
 548
 549        hscb = scb->hscb;
 550        /* Clean up for the next user */
 551        ahc->scb_data->scbindex[hscb->tag] = NULL;
 552        scb->flags = SCB_FREE;
 553        hscb->control = 0;
 554
 555        SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs, scb, links.sle);
 556
 557        /* Notify the OSM that a resource is now available. */
 558        ahc_platform_scb_free(ahc, scb);
 559}
 560
 561struct scb *
 562ahc_lookup_scb(struct ahc_softc *ahc, u_int tag)
 563{
 564        struct scb* scb;
 565
 566        scb = ahc->scb_data->scbindex[tag];
 567        if (scb != NULL)
 568                ahc_sync_scb(ahc, scb,
 569                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 570        return (scb);
 571}
 572
 573static void
 574ahc_swap_with_next_hscb(struct ahc_softc *ahc, struct scb *scb)
 575{
 576        struct hardware_scb *q_hscb;
 577        u_int  saved_tag;
 578
 579        /*
 580         * Our queuing method is a bit tricky.  The card
 581         * knows in advance which HSCB to download, and we
 582         * can't disappoint it.  To achieve this, the next
 583         * SCB to download is saved off in ahc->next_queued_scb.
 584         * When we are called to queue "an arbitrary scb",
 585         * we copy the contents of the incoming HSCB to the one
 586         * the sequencer knows about, swap HSCB pointers and
 587         * finally assign the SCB to the tag indexed location
 588         * in the scb_array.  This makes sure that we can still
 589         * locate the correct SCB by SCB_TAG.
 590         */
 591        q_hscb = ahc->next_queued_scb->hscb;
 592        saved_tag = q_hscb->tag;
 593        memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 594        if ((scb->flags & SCB_CDB32_PTR) != 0) {
 595                q_hscb->shared_data.cdb_ptr =
 596                    ahc_htole32(ahc_hscb_busaddr(ahc, q_hscb->tag)
 597                              + offsetof(struct hardware_scb, cdb32));
 598        }
 599        q_hscb->tag = saved_tag;
 600        q_hscb->next = scb->hscb->tag;
 601
 602        /* Now swap HSCB pointers. */
 603        ahc->next_queued_scb->hscb = scb->hscb;
 604        scb->hscb = q_hscb;
 605
 606        /* Now define the mapping from tag to SCB in the scbindex */
 607        ahc->scb_data->scbindex[scb->hscb->tag] = scb;
 608}
 609
 610/*
 611 * Tell the sequencer about a new transaction to execute.
 612 */
 613void
 614ahc_queue_scb(struct ahc_softc *ahc, struct scb *scb)
 615{
 616        ahc_swap_with_next_hscb(ahc, scb);
 617
 618        if (scb->hscb->tag == SCB_LIST_NULL
 619         || scb->hscb->next == SCB_LIST_NULL)
 620                panic("Attempt to queue invalid SCB tag %x:%x\n",
 621                      scb->hscb->tag, scb->hscb->next);
 622
 623        /*
 624         * Setup data "oddness".
 625         */
 626        scb->hscb->lun &= LID;
 627        if (ahc_get_transfer_length(scb) & 0x1)
 628                scb->hscb->lun |= SCB_XFERLEN_ODD;
 629
 630        /*
 631         * Keep a history of SCBs we've downloaded in the qinfifo.
 632         */
 633        ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
 634
 635        /*
 636         * Make sure our data is consistent from the
 637         * perspective of the adapter.
 638         */
 639        ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 640
 641        /* Tell the adapter about the newly queued SCB */
 642        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
 643                ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
 644        } else {
 645                if ((ahc->features & AHC_AUTOPAUSE) == 0)
 646                        ahc_pause(ahc);
 647                ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
 648                if ((ahc->features & AHC_AUTOPAUSE) == 0)
 649                        ahc_unpause(ahc);
 650        }
 651}
 652
 653struct scsi_sense_data *
 654ahc_get_sense_buf(struct ahc_softc *ahc, struct scb *scb)
 655{
 656        int offset;
 657
 658        offset = scb - ahc->scb_data->scbarray;
 659        return (&ahc->scb_data->sense[offset]);
 660}
 661
 662static uint32_t
 663ahc_get_sense_bufaddr(struct ahc_softc *ahc, struct scb *scb)
 664{
 665        int offset;
 666
 667        offset = scb - ahc->scb_data->scbarray;
 668        return (ahc->scb_data->sense_busaddr
 669              + (offset * sizeof(struct scsi_sense_data)));
 670}
 671
 672/************************** Interrupt Processing ******************************/
 673static void
 674ahc_sync_qoutfifo(struct ahc_softc *ahc, int op)
 675{
 676        ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 677                        /*offset*/0, /*len*/256, op);
 678}
 679
 680static void
 681ahc_sync_tqinfifo(struct ahc_softc *ahc, int op)
 682{
 683#ifdef AHC_TARGET_MODE
 684        if ((ahc->flags & AHC_TARGETROLE) != 0) {
 685                ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 686                                ahc->shared_data_dmamap,
 687                                ahc_targetcmd_offset(ahc, 0),
 688                                sizeof(struct target_cmd) * AHC_TMODE_CMDS,
 689                                op);
 690        }
 691#endif
 692}
 693
 694/*
 695 * See if the firmware has posted any completed commands
 696 * into our in-core command complete fifos.
 697 */
 698#define AHC_RUN_QOUTFIFO 0x1
 699#define AHC_RUN_TQINFIFO 0x2
 700static u_int
 701ahc_check_cmdcmpltqueues(struct ahc_softc *ahc)
 702{
 703        u_int retval;
 704
 705        retval = 0;
 706        ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 707                        /*offset*/ahc->qoutfifonext, /*len*/1,
 708                        BUS_DMASYNC_POSTREAD);
 709        if (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL)
 710                retval |= AHC_RUN_QOUTFIFO;
 711#ifdef AHC_TARGET_MODE
 712        if ((ahc->flags & AHC_TARGETROLE) != 0
 713         && (ahc->flags & AHC_TQINFIFO_BLOCKED) == 0) {
 714                ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 715                                ahc->shared_data_dmamap,
 716                                ahc_targetcmd_offset(ahc, ahc->tqinfifofnext),
 717                                /*len*/sizeof(struct target_cmd),
 718                                BUS_DMASYNC_POSTREAD);
 719                if (ahc->targetcmds[ahc->tqinfifonext].cmd_valid != 0)
 720                        retval |= AHC_RUN_TQINFIFO;
 721        }
 722#endif
 723        return (retval);
 724}
 725
 726/*
 727 * Catch an interrupt from the adapter
 728 */
 729int
 730ahc_intr(struct ahc_softc *ahc)
 731{
 732        u_int        intstat;
 733
 734        if ((ahc->pause & INTEN) == 0) {
 735                /*
 736                 * Our interrupt is not enabled on the chip
 737                 * and may be disabled for re-entrancy reasons,
 738                 * so just return.  This is likely just a shared
 739                 * interrupt.
 740                 */
 741                return (0);
 742        }
 743        /*
 744         * Instead of directly reading the interrupt status register,
 745         * infer the cause of the interrupt by checking our in-core
 746         * completion queues.  This avoids a costly PCI bus read in
 747         * most cases.
 748         */
 749        if ((ahc->flags & (AHC_ALL_INTERRUPTS|AHC_EDGE_INTERRUPT)) == 0
 750         && (ahc_check_cmdcmpltqueues(ahc) != 0))
 751                intstat = CMDCMPLT;
 752        else {
 753                intstat = ahc_inb(ahc, INTSTAT);
 754        }
 755
 756        if ((intstat & INT_PEND) == 0) {
 757#if AHC_PCI_CONFIG > 0
 758                if (ahc->unsolicited_ints > 500) {
 759                        ahc->unsolicited_ints = 0;
 760                        if ((ahc->chip & AHC_PCI) != 0
 761                         && (ahc_inb(ahc, ERROR) & PCIERRSTAT) != 0)
 762                                ahc->bus_intr(ahc);
 763                }
 764#endif
 765                ahc->unsolicited_ints++;
 766                return (0);
 767        }
 768        ahc->unsolicited_ints = 0;
 769
 770        if (intstat & CMDCMPLT) {
 771                ahc_outb(ahc, CLRINT, CLRCMDINT);
 772
 773                /*
 774                 * Ensure that the chip sees that we've cleared
 775                 * this interrupt before we walk the output fifo.
 776                 * Otherwise, we may, due to posted bus writes,
 777                 * clear the interrupt after we finish the scan,
 778                 * and after the sequencer has added new entries
 779                 * and asserted the interrupt again.
 780                 */
 781                ahc_flush_device_writes(ahc);
 782                ahc_run_qoutfifo(ahc);
 783#ifdef AHC_TARGET_MODE
 784                if ((ahc->flags & AHC_TARGETROLE) != 0)
 785                        ahc_run_tqinfifo(ahc, /*paused*/FALSE);
 786#endif
 787        }
 788
 789        /*
 790         * Handle statuses that may invalidate our cached
 791         * copy of INTSTAT separately.
 792         */
 793        if (intstat == 0xFF && (ahc->features & AHC_REMOVABLE) != 0) {
 794                /* Hot eject.  Do nothing */
 795        } else if (intstat & BRKADRINT) {
 796                ahc_handle_brkadrint(ahc);
 797        } else if ((intstat & (SEQINT|SCSIINT)) != 0) {
 798
 799                ahc_pause_bug_fix(ahc);
 800
 801                if ((intstat & SEQINT) != 0)
 802                        ahc_handle_seqint(ahc, intstat);
 803
 804                if ((intstat & SCSIINT) != 0)
 805                        ahc_handle_scsiint(ahc, intstat);
 806        }
 807        return (1);
 808}
 809
 810/************************* Sequencer Execution Control ************************/
 811/*
 812 * Restart the sequencer program from address zero
 813 */
 814static void
 815ahc_restart(struct ahc_softc *ahc)
 816{
 817        uint8_t        sblkctl;
 818
 819        ahc_pause(ahc);
 820
 821        /* No more pending messages. */
 822        ahc_clear_msg_state(ahc);
 823
 824        ahc_outb(ahc, SCSISIGO, 0);                /* De-assert BSY */
 825        ahc_outb(ahc, MSG_OUT, MSG_NOOP);        /* No message to send */
 826        ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 827        ahc_outb(ahc, LASTPHASE, P_BUSFREE);
 828        ahc_outb(ahc, SAVED_SCSIID, 0xFF);
 829        ahc_outb(ahc, SAVED_LUN, 0xFF);
 830
 831        /*
 832         * Ensure that the sequencer's idea of TQINPOS
 833         * matches our own.  The sequencer increments TQINPOS
 834         * only after it sees a DMA complete and a reset could
 835         * occur before the increment leaving the kernel to believe
 836         * the command arrived but the sequencer to not.
 837         */
 838        ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
 839
 840        /* Always allow reselection */
 841        ahc_outb(ahc, SCSISEQ,
 842                 ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 843        if ((ahc->features & AHC_CMD_CHAN) != 0) {
 844                /* Ensure that no DMA operations are in progress */
 845                ahc_outb(ahc, CCSCBCNT, 0);
 846                ahc_outb(ahc, CCSGCTL, 0);
 847                ahc_outb(ahc, CCSCBCTL, 0);
 848        }
 849        /*
 850         * If we were in the process of DMA'ing SCB data into
 851         * an SCB, replace that SCB on the free list.  This prevents
 852         * an SCB leak.
 853         */
 854        if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
 855                ahc_add_curscb_to_free_list(ahc);
 856                ahc_outb(ahc, SEQ_FLAGS2,
 857                         ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
 858        }
 859
 860        /*
 861         * Clear any pending sequencer interrupt.  It is no
 862         * longer relevant since we're resetting the Program
 863         * Counter.
 864         */
 865        ahc_outb(ahc, CLRINT, CLRSEQINT);
 866
 867        ahc_outb(ahc, MWI_RESIDUAL, 0);
 868        ahc_outb(ahc, SEQCTL, ahc->seqctl);
 869        ahc_outb(ahc, SEQADDR0, 0);
 870        ahc_outb(ahc, SEQADDR1, 0);
 871
 872        /*
 873         * Take the LED out of diagnostic mode on PM resume, too
 874         */
 875        sblkctl = ahc_inb(ahc, SBLKCTL);
 876        ahc_outb(ahc, SBLKCTL, (sblkctl & ~(DIAGLEDEN|DIAGLEDON)));
 877
 878        ahc_unpause(ahc);
 879}
 880
 881/************************* Input/Output Queues ********************************/
 882static void
 883ahc_run_qoutfifo(struct ahc_softc *ahc)
 884{
 885        struct scb *scb;
 886        u_int  scb_index;
 887
 888        ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
 889        while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
 890
 891                scb_index = ahc->qoutfifo[ahc->qoutfifonext];
 892                if ((ahc->qoutfifonext & 0x03) == 0x03) {
 893                        u_int modnext;
 894
 895                        /*
 896                         * Clear 32bits of QOUTFIFO at a time
 897                         * so that we don't clobber an incoming
 898                         * byte DMA to the array on architectures
 899                         * that only support 32bit load and store
 900                         * operations.
 901                         */
 902                        modnext = ahc->qoutfifonext & ~0x3;
 903                        *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
 904                        ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 905                                        ahc->shared_data_dmamap,
 906                                        /*offset*/modnext, /*len*/4,
 907                                        BUS_DMASYNC_PREREAD);
 908                }
 909                ahc->qoutfifonext++;
 910
 911                scb = ahc_lookup_scb(ahc, scb_index);
 912                if (scb == NULL) {
 913                        printf("%s: WARNING no command for scb %d "
 914                               "(cmdcmplt)\nQOUTPOS = %d\n",
 915                               ahc_name(ahc), scb_index,
 916                               (ahc->qoutfifonext - 1) & 0xFF);
 917                        continue;
 918                }
 919
 920                /*
 921                 * Save off the residual
 922                 * if there is one.
 923                 */
 924                ahc_update_residual(ahc, scb);
 925                ahc_done(ahc, scb);
 926        }
 927}
 928
 929static void
 930ahc_run_untagged_queues(struct ahc_softc *ahc)
 931{
 932        int i;
 933
 934        for (i = 0; i < 16; i++)
 935                ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
 936}
 937
 938static void
 939ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
 940{
 941        struct scb *scb;
 942
 943        if (ahc->untagged_queue_lock != 0)
 944                return;
 945
 946        if ((scb = TAILQ_FIRST(queue)) != NULL
 947         && (scb->flags & SCB_ACTIVE) == 0) {
 948                scb->flags |= SCB_ACTIVE;
 949                ahc_queue_scb(ahc, scb);
 950        }
 951}
 952
 953/************************* Interrupt Handling *********************************/
 954static void
 955ahc_handle_brkadrint(struct ahc_softc *ahc)
 956{
 957        /*
 958         * We upset the sequencer :-(
 959         * Lookup the error message
 960         */
 961        int i;
 962        int error;
 963
 964        error = ahc_inb(ahc, ERROR);
 965        for (i = 0; error != 1 && i < num_errors; i++)
 966                error >>= 1;
 967        printf("%s: brkadrint, %s at seqaddr = 0x%x\n",
 968               ahc_name(ahc), ahc_hard_errors[i].errmesg,
 969               ahc_inb(ahc, SEQADDR0) |
 970               (ahc_inb(ahc, SEQADDR1) << 8));
 971
 972        ahc_dump_card_state(ahc);
 973
 974        /* Tell everyone that this HBA is no longer available */
 975        ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 976                       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 977                       CAM_NO_HBA);
 978
 979        /* Disable all interrupt sources by resetting the controller */
 980        ahc_shutdown(ahc);
 981}
 982
 983static void
 984ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
 985{
 986        struct scb *scb;
 987        struct ahc_devinfo devinfo;
 988        
 989        ahc_fetch_devinfo(ahc, &devinfo);
 990
 991        /*
 992         * Clear the upper byte that holds SEQINT status
 993         * codes and clear the SEQINT bit. We will unpause
 994         * the sequencer, if appropriate, after servicing
 995         * the request.
 996         */
 997        ahc_outb(ahc, CLRINT, CLRSEQINT);
 998        switch (intstat & SEQINT_MASK) {
 999        case BAD_STATUS:
1000        {
1001                u_int  scb_index;
1002                struct hardware_scb *hscb;
1003
1004                /*
1005                 * Set the default return value to 0 (don't
1006                 * send sense).  The sense code will change
1007                 * this if needed.
1008                 */
1009                ahc_outb(ahc, RETURN_1, 0);
1010
1011                /*
1012                 * The sequencer will notify us when a command
1013                 * has an error that would be of interest to
1014                 * the kernel.  This allows us to leave the sequencer
1015                 * running in the common case of command completes
1016                 * without error.  The sequencer will already have
1017                 * dma'd the SCB back up to us, so we can reference
1018                 * the in kernel copy directly.
1019                 */
1020                scb_index = ahc_inb(ahc, SCB_TAG);
1021                scb = ahc_lookup_scb(ahc, scb_index);
1022                if (scb == NULL) {
1023                        ahc_print_devinfo(ahc, &devinfo);
1024                        printf("ahc_intr - referenced scb "
1025                               "not valid during seqint 0x%x scb(%d)\n",
1026                               intstat, scb_index);
1027                        ahc_dump_card_state(ahc);
1028                        panic("for safety");
1029                        goto unpause;
1030                }
1031
1032                hscb = scb->hscb; 
1033
1034                /* Don't want to clobber the original sense code */
1035                if ((scb->flags & SCB_SENSE) != 0) {
1036                        /*
1037                         * Clear the SCB_SENSE Flag and have
1038                         * the sequencer do a normal command
1039                         * complete.
1040                         */
1041                        scb->flags &= ~SCB_SENSE;
1042                        ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1043                        break;
1044                }
1045                ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
1046                /* Freeze the queue until the client sees the error. */
1047                ahc_freeze_devq(ahc, scb);
1048                ahc_freeze_scb(scb);
1049                ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
1050                switch (hscb->shared_data.status.scsi_status) {
1051                case SCSI_STATUS_OK:
1052                        printf("%s: Interrupted for staus of 0???\n",
1053                               ahc_name(ahc));
1054                        break;
1055                case SCSI_STATUS_CMD_TERMINATED:
1056                case SCSI_STATUS_CHECK_COND:
1057                {
1058                        struct ahc_dma_seg *sg;
1059                        struct scsi_sense *sc;
1060                        struct ahc_initiator_tinfo *targ_info;
1061                        struct ahc_tmode_tstate *tstate;
1062                        struct ahc_transinfo *tinfo;
1063#ifdef AHC_DEBUG
1064                        if (ahc_debug & AHC_SHOW_SENSE) {
1065                                ahc_print_path(ahc, scb);
1066                                printf("SCB %d: requests Check Status\n",
1067                                       scb->hscb->tag);
1068                        }
1069#endif
1070
1071                        if (ahc_perform_autosense(scb) == 0)
1072                                break;
1073
1074                        targ_info = ahc_fetch_transinfo(ahc,
1075                                                        devinfo.channel,
1076                                                        devinfo.our_scsiid,
1077                                                        devinfo.target,
1078                                                        &tstate);
1079                        tinfo = &targ_info->curr;
1080                        sg = scb->sg_list;
1081                        sc = (struct scsi_sense *)(&hscb->shared_data.cdb); 
1082                        /*
1083                         * Save off the residual if there is one.
1084                         */
1085                        ahc_update_residual(ahc, scb);
1086#ifdef AHC_DEBUG
1087                        if (ahc_debug & AHC_SHOW_SENSE) {
1088                                ahc_print_path(ahc, scb);
1089                                printf("Sending Sense\n");
1090                        }
1091#endif
1092                        sg->addr = ahc_get_sense_bufaddr(ahc, scb);
1093                        sg->len = ahc_get_sense_bufsize(ahc, scb);
1094                        sg->len |= AHC_DMA_LAST_SEG;
1095
1096                        /* Fixup byte order */
1097                        sg->addr = ahc_htole32(sg->addr);
1098                        sg->len = ahc_htole32(sg->len);
1099
1100                        sc->opcode = REQUEST_SENSE;
1101                        sc->byte2 = 0;
1102                        if (tinfo->protocol_version <= SCSI_REV_2
1103                         && SCB_GET_LUN(scb) < 8)
1104                                sc->byte2 = SCB_GET_LUN(scb) << 5;
1105                        sc->unused[0] = 0;
1106                        sc->unused[1] = 0;
1107                        sc->length = sg->len;
1108                        sc->control = 0;
1109
1110                        /*
1111                         * We can't allow the target to disconnect.
1112                         * This will be an untagged transaction and
1113                         * having the target disconnect will make this
1114                         * transaction indestinguishable from outstanding
1115                         * tagged transactions.
1116                         */
1117                        hscb->control = 0;
1118
1119                        /*
1120                         * This request sense could be because the
1121                         * the device lost power or in some other
1122                         * way has lost our transfer negotiations.
1123                         * Renegotiate if appropriate.  Unit attention
1124                         * errors will be reported before any data
1125                         * phases occur.
1126                         */
1127                        if (ahc_get_residual(scb) 
1128                         == ahc_get_transfer_length(scb)) {
1129                                ahc_update_neg_request(ahc, &devinfo,
1130                                                       tstate, targ_info,
1131                                                       AHC_NEG_IF_NON_ASYNC);
1132                        }
1133                        if (tstate->auto_negotiate & devinfo.target_mask) {
1134                                hscb->control |= MK_MESSAGE;
1135                                scb->flags &= ~SCB_NEGOTIATE;
1136                                scb->flags |= SCB_AUTO_NEGOTIATE;
1137                        }
1138                        hscb->cdb_len = sizeof(*sc);
1139                        hscb->dataptr = sg->addr; 
1140                        hscb->datacnt = sg->len;
1141                        hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
1142                        hscb->sgptr = ahc_htole32(hscb->sgptr);
1143                        scb->sg_count = 1;
1144                        scb->flags |= SCB_SENSE;
1145                        ahc_qinfifo_requeue_tail(ahc, scb);
1146                        ahc_outb(ahc, RETURN_1, SEND_SENSE);
1147                        /*
1148                         * Ensure we have enough time to actually
1149                         * retrieve the sense.
1150                         */
1151                        ahc_scb_timer_reset(scb, 5 * 1000000);
1152                        break;
1153                }
1154                default:
1155                        break;
1156                }
1157                break;
1158        }
1159        case NO_MATCH:
1160        {
1161                /* Ensure we don't leave the selection hardware on */
1162                ahc_outb(ahc, SCSISEQ,
1163                         ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1164
1165                printf("%s:%c:%d: no active SCB for reconnecting "
1166                       "target - issuing BUS DEVICE RESET\n",
1167                       ahc_name(ahc), devinfo.channel, devinfo.target);
1168                printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1169                       "ARG_1 == 0x%x ACCUM = 0x%x\n",
1170                       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1171                       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1172                printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1173                       "SINDEX == 0x%x\n",
1174                       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1175                       ahc_index_busy_tcl(ahc,
1176                            BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1177                                      ahc_inb(ahc, SAVED_LUN))),
1178                       ahc_inb(ahc, SINDEX));
1179                printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1180                       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1181                       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1182                       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1183                       ahc_inb(ahc, SCB_CONTROL));
1184                printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1185                       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1186                printf("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
1187                printf("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
1188                ahc_dump_card_state(ahc);
1189                ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
1190                ahc->msgout_len = 1;
1191                ahc->msgout_index = 0;
1192                ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1193                ahc_outb(ahc, MSG_OUT, HOST_MSG);
1194                ahc_assert_atn(ahc);
1195                break;
1196        }
1197        case SEND_REJECT: 
1198        {
1199                u_int rejbyte = ahc_inb(ahc, ACCUM);
1200                printf("%s:%c:%d: Warning - unknown message received from "
1201                       "target (0x%x).  Rejecting\n", 
1202                       ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
1203                break; 
1204        }
1205        case PROTO_VIOLATION:
1206        {
1207                ahc_handle_proto_violation(ahc);
1208                break;
1209        }
1210        case IGN_WIDE_RES:
1211                ahc_handle_ign_wide_residue(ahc, &devinfo);
1212                break;
1213        case PDATA_REINIT:
1214                ahc_reinitialize_dataptrs(ahc);
1215                break;
1216        case BAD_PHASE:
1217        {
1218                u_int lastphase;
1219
1220                lastphase = ahc_inb(ahc, LASTPHASE);
1221                printf("%s:%c:%d: unknown scsi bus phase %x, "
1222                       "lastphase = 0x%x.  Attempting to continue\n",
1223                       ahc_name(ahc), devinfo.channel, devinfo.target,
1224                       lastphase, ahc_inb(ahc, SCSISIGI));
1225                break;
1226        }
1227        case MISSED_BUSFREE:
1228        {
1229                u_int lastphase;
1230
1231                lastphase = ahc_inb(ahc, LASTPHASE);
1232                printf("%s:%c:%d: Missed busfree. "
1233                       "Lastphase = 0x%x, Curphase = 0x%x\n",
1234                       ahc_name(ahc), devinfo.channel, devinfo.target,
1235                       lastphase, ahc_inb(ahc, SCSISIGI));
1236                ahc_restart(ahc);
1237                return;
1238        }
1239        case HOST_MSG_LOOP:
1240        {
1241                /*
1242                 * The sequencer has encountered a message phase
1243                 * that requires host assistance for completion.
1244                 * While handling the message phase(s), we will be
1245                 * notified by the sequencer after each byte is
1246                 * transfered so we can track bus phase changes.
1247                 *
1248                 * If this is the first time we've seen a HOST_MSG_LOOP
1249                 * interrupt, initialize the state of the host message
1250                 * loop.
1251                 */
1252                if (ahc->msg_type == MSG_TYPE_NONE) {
1253                        struct scb *scb;
1254                        u_int scb_index;
1255                        u_int bus_phase;
1256
1257                        bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1258                        if (bus_phase != P_MESGIN
1259                         && bus_phase != P_MESGOUT) {
1260                                printf("ahc_intr: HOST_MSG_LOOP bad "
1261                                       "phase 0x%x\n",
1262                                      bus_phase);
1263                                /*
1264                                 * Probably transitioned to bus free before
1265                                 * we got here.  Just punt the message.
1266                                 */
1267                                ahc_clear_intstat(ahc);
1268                                ahc_restart(ahc);
1269                                return;
1270                        }
1271
1272                        scb_index = ahc_inb(ahc, SCB_TAG);
1273                        scb = ahc_lookup_scb(ahc, scb_index);
1274                        if (devinfo.role == ROLE_INITIATOR) {
1275                                if (bus_phase == P_MESGOUT) {
1276                                        if (scb == NULL)
1277                                                panic("HOST_MSG_LOOP with "
1278                                                      "invalid SCB %x\n",
1279                                                      scb_index);
1280
1281                                        ahc_setup_initiator_msgout(ahc,
1282                                                                   &devinfo,
1283                                                                   scb);
1284                                } else {
1285                                        ahc->msg_type =
1286                                            MSG_TYPE_INITIATOR_MSGIN;
1287                                        ahc->msgin_index = 0;
1288                                }
1289                        }
1290#ifdef AHC_TARGET_MODE
1291                        else {
1292                                if (bus_phase == P_MESGOUT) {
1293                                        ahc->msg_type =
1294                                            MSG_TYPE_TARGET_MSGOUT;
1295                                        ahc->msgin_index = 0;
1296                                }
1297                                else 
1298                                        ahc_setup_target_msgin(ahc,
1299                                                               &devinfo,
1300                                                               scb);
1301                        }
1302#endif
1303                }
1304
1305                ahc_handle_message_phase(ahc);
1306                break;
1307        }
1308        case PERR_DETECTED:
1309        {
1310                /*
1311                 * If we've cleared the parity error interrupt
1312                 * but the sequencer still believes that SCSIPERR
1313                 * is true, it must be that the parity error is
1314                 * for the currently presented byte on the bus,
1315                 * and we are not in a phase (data-in) where we will
1316                 * eventually ack this byte.  Ack the byte and
1317                 * throw it away in the hope that the target will
1318                 * take us to message out to deliver the appropriate
1319                 * error message.
1320                 */
1321                if ((intstat & SCSIINT) == 0
1322                 && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
1323
1324                        if ((ahc->features & AHC_DT) == 0) {
1325                                u_int curphase;
1326
1327                                /*
1328                                 * The hardware will only let you ack bytes
1329                                 * if the expected phase in SCSISIGO matches
1330                                 * the current phase.  Make sure this is
1331                                 * currently the case.
1332                                 */
1333                                curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1334                                ahc_outb(ahc, LASTPHASE, curphase);
1335                                ahc_outb(ahc, SCSISIGO, curphase);
1336                        }
1337                        if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
1338                                int wait;
1339
1340                                /*
1341                                 * In a data phase.  Faster to bitbucket
1342                                 * the data than to individually ack each
1343                                 * byte.  This is also the only strategy
1344                                 * that will work with AUTOACK enabled.
1345                                 */
1346                                ahc_outb(ahc, SXFRCTL1,
1347                                         ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
1348                                wait = 5000;
1349                                while (--wait != 0) {
1350                                        if ((ahc_inb(ahc, SCSISIGI)
1351                                          & (CDI|MSGI)) != 0)
1352                                                break;
1353                                        ahc_delay(100);
1354                                }
1355                                ahc_outb(ahc, SXFRCTL1,
1356                                         ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
1357                                if (wait == 0) {
1358                                        struct        scb *scb;
1359                                        u_int        scb_index;
1360
1361                                        ahc_print_devinfo(ahc, &devinfo);
1362                                        printf("Unable to clear parity error.  "
1363                                               "Resetting bus.\n");
1364                                        scb_index = ahc_inb(ahc, SCB_TAG);
1365                                        scb = ahc_lookup_scb(ahc, scb_index);
1366                                        if (scb != NULL)
1367                                                ahc_set_transaction_status(scb,
1368                                                    CAM_UNCOR_PARITY);
1369                                        ahc_reset_channel(ahc, devinfo.channel, 
1370                                                          /*init reset*/TRUE);
1371                                }
1372                        } else {
1373                                ahc_inb(ahc, SCSIDATL);
1374                        }
1375                }
1376                break;
1377        }
1378        case DATA_OVERRUN:
1379        {
1380                /*
1381                 * When the sequencer detects an overrun, it
1382                 * places the controller in "BITBUCKET" mode
1383                 * and allows the target to complete its transfer.
1384                 * Unfortunately, none of the counters get updated
1385                 * when the controller is in this mode, so we have
1386                 * no way of knowing how large the overrun was.
1387                 */
1388                u_int scbindex = ahc_inb(ahc, SCB_TAG);
1389                u_int lastphase = ahc_inb(ahc, LASTPHASE);
1390                u_int i;
1391
1392                scb = ahc_lookup_scb(ahc, scbindex);
1393                for (i = 0; i < num_phases; i++) {
1394                        if (lastphase == ahc_phase_table[i].phase)
1395                                break;
1396                }
1397                ahc_print_path(ahc, scb);
1398                printf("data overrun detected %s."
1399                       "  Tag == 0x%x.\n",
1400                       ahc_phase_table[i].phasemsg,
1401                         scb->hscb->tag);
1402                ahc_print_path(ahc, scb);
1403                printf("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
1404                       ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
1405                       ahc_get_transfer_length(scb), scb->sg_count);
1406                if (scb->sg_count > 0) {
1407                        for (i = 0; i < scb->sg_count; i++) {
1408
1409                                printf("sg[%d] - Addr 0x%x%x : Length %d\n",
1410                                       i,
1411                                       (ahc_le32toh(scb->sg_list[i].len) >> 24
1412                                        & SG_HIGH_ADDR_BITS),
1413                                       ahc_le32toh(scb->sg_list[i].addr),
1414                                       ahc_le32toh(scb->sg_list[i].len)
1415                                       & AHC_SG_LEN_MASK);
1416                        }
1417                }
1418                /*
1419                 * Set this and it will take effect when the
1420                 * target does a command complete.
1421                 */
1422                ahc_freeze_devq(ahc, scb);
1423                if ((scb->flags & SCB_SENSE) == 0) {
1424                        ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1425                } else {
1426                        scb->flags &= ~SCB_SENSE;
1427                        ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1428                }
1429                ahc_freeze_scb(scb);
1430
1431                if ((ahc->features & AHC_ULTRA2) != 0) {
1432                        /*
1433                         * Clear the channel in case we return
1434                         * to data phase later.
1435                         */
1436                        ahc_outb(ahc, SXFRCTL0,
1437                                 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1438                        ahc_outb(ahc, SXFRCTL0,
1439                                 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1440                }
1441                if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
1442                        u_int dscommand1;
1443
1444                        /* Ensure HHADDR is 0 for future DMA operations. */
1445                        dscommand1 = ahc_inb(ahc, DSCOMMAND1);
1446                        ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
1447                        ahc_outb(ahc, HADDR, 0);
1448                        ahc_outb(ahc, DSCOMMAND1, dscommand1);
1449                }
1450                break;
1451        }
1452        case MKMSG_FAILED:
1453        {
1454                u_int scbindex;
1455
1456                printf("%s:%c:%d:%d: Attempt to issue message failed\n",
1457                       ahc_name(ahc), devinfo.channel, devinfo.target,
1458                       devinfo.lun);
1459                scbindex = ahc_inb(ahc, SCB_TAG);
1460                scb = ahc_lookup_scb(ahc, scbindex);
1461                if (scb != NULL
1462                 && (scb->flags & SCB_RECOVERY_SCB) != 0)
1463                        /*
1464                         * Ensure that we didn't put a second instance of this
1465                         * SCB into the QINFIFO.
1466                         */
1467                        ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
1468                                           SCB_GET_CHANNEL(ahc, scb),
1469                                           SCB_GET_LUN(scb), scb->hscb->tag,
1470                                           ROLE_INITIATOR, /*status*/0,
1471                                           SEARCH_REMOVE);
1472                break;
1473        }
1474        case NO_FREE_SCB:
1475        {
1476                printf("%s: No free or disconnected SCBs\n", ahc_name(ahc));
1477                ahc_dump_card_state(ahc);
1478                panic("for safety");
1479                break;
1480        }
1481        case SCB_MISMATCH:
1482        {
1483                u_int scbptr;
1484
1485                scbptr = ahc_inb(ahc, SCBPTR);
1486                printf("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
1487                       scbptr, ahc_inb(ahc, ARG_1),
1488                       ahc->scb_data->hscbs[scbptr].tag);
1489                ahc_dump_card_state(ahc);
1490                panic("for saftey");
1491                break;
1492        }
1493        case OUT_OF_RANGE:
1494        {
1495                printf("%s: BTT calculation out of range\n", ahc_name(ahc));
1496                printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1497                       "ARG_1 == 0x%x ACCUM = 0x%x\n",
1498                       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1499                       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1500                printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1501                       "SINDEX == 0x%x\n, A == 0x%x\n",
1502                       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1503                       ahc_index_busy_tcl(ahc,
1504                            BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1505                                      ahc_inb(ahc, SAVED_LUN))),
1506                       ahc_inb(ahc, SINDEX),
1507                       ahc_inb(ahc, ACCUM));
1508                printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1509                       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1510                       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1511                       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1512                       ahc_inb(ahc, SCB_CONTROL));
1513                printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1514                       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1515                ahc_dump_card_state(ahc);
1516                panic("for safety");
1517                break;
1518        }
1519        default:
1520                printf("ahc_intr: seqint, "
1521                       "intstat == 0x%x, scsisigi = 0x%x\n",
1522                       intstat, ahc_inb(ahc, SCSISIGI));
1523                break;
1524        }
1525unpause:
1526        /*
1527         *  The sequencer is paused immediately on
1528         *  a SEQINT, so we should restart it when
1529         *  we're done.
1530         */
1531        ahc_unpause(ahc);
1532}
1533
1534static void
1535ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
1536{
1537        u_int        scb_index;
1538        u_int        status0;
1539        u_int        status;
1540        struct        scb *scb;
1541        char        cur_channel;
1542        char        intr_channel;
1543
1544        if ((ahc->features & AHC_TWIN) != 0
1545         && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
1546                cur_channel = 'B';
1547        else
1548                cur_channel = 'A';
1549        intr_channel = cur_channel;
1550
1551        if ((ahc->features & AHC_ULTRA2) != 0)
1552                status0 = ahc_inb(ahc, SSTAT0) & IOERR;
1553        else
1554                status0 = 0;
1555        status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1556        if (status == 0 && status0 == 0) {
1557                if ((ahc->features & AHC_TWIN) != 0) {
1558                        /* Try the other channel */
1559                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1560                        status = ahc_inb(ahc, SSTAT1)
1561                               & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1562                        intr_channel = (cur_channel == 'A') ? 'B' : 'A';
1563                }
1564                if (status == 0) {
1565                        printf("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
1566                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
1567                        ahc_unpause(ahc);
1568                        return;
1569                }
1570        }
1571
1572        /* Make sure the sequencer is in a safe location. */
1573        ahc_clear_critical_section(ahc);
1574
1575        scb_index = ahc_inb(ahc, SCB_TAG);
1576        scb = ahc_lookup_scb(ahc, scb_index);
1577        if (scb != NULL
1578         && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1579                scb = NULL;
1580
1581        if ((ahc->features & AHC_ULTRA2) != 0
1582         && (status0 & IOERR) != 0) {
1583                int now_lvd;
1584
1585                now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
1586                printf("%s: Transceiver State Has Changed to %s mode\n",
1587                       ahc_name(ahc), now_lvd ? "LVD" : "SE");
1588                ahc_outb(ahc, CLRSINT0, CLRIOERR);
1589                /*
1590                 * When transitioning to SE mode, the reset line
1591                 * glitches, triggering an arbitration bug in some
1592                 * Ultra2 controllers.  This bug is cleared when we
1593                 * assert the reset line.  Since a reset glitch has
1594                 * already occurred with this transition and a
1595                 * transceiver state change is handled just like
1596                 * a bus reset anyway, asserting the reset line
1597                 * ourselves is safe.
1598                 */
1599                ahc_reset_channel(ahc, intr_channel,
1600                                 /*Initiate Reset*/now_lvd == 0);
1601        } else if ((status & SCSIRSTI) != 0) {
1602                printf("%s: Someone reset channel %c\n",
1603                        ahc_name(ahc), intr_channel);
1604                if (intr_channel != cur_channel)
1605                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1606                ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1607        } else if ((status & SCSIPERR) != 0) {
1608                /*
1609                 * Determine the bus phase and queue an appropriate message.
1610                 * SCSIPERR is latched true as soon as a parity error
1611                 * occurs.  If the sequencer acked the transfer that
1612                 * caused the parity error and the currently presented
1613                 * transfer on the bus has correct parity, SCSIPERR will
1614                 * be cleared by CLRSCSIPERR.  Use this to determine if
1615                 * we should look at the last phase the sequencer recorded,
1616                 * or the current phase presented on the bus.
1617                 */
1618                struct        ahc_devinfo devinfo;
1619                u_int        mesg_out;
1620                u_int        curphase;
1621                u_int        errorphase;
1622                u_int        lastphase;
1623                u_int        scsirate;
1624                u_int        i;
1625                u_int        sstat2;
1626                int        silent;
1627
1628                lastphase = ahc_inb(ahc, LASTPHASE);
1629                curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1630                sstat2 = ahc_inb(ahc, SSTAT2);
1631                ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1632                /*
1633                 * For all phases save DATA, the sequencer won't
1634                 * automatically ack a byte that has a parity error
1635                 * in it.  So the only way that the current phase
1636                 * could be 'data-in' is if the parity error is for
1637                 * an already acked byte in the data phase.  During
1638                 * synchronous data-in transfers, we may actually
1639                 * ack bytes before latching the current phase in
1640                 * LASTPHASE, leading to the discrepancy between
1641                 * curphase and lastphase.
1642                 */
1643                if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1644                 || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1645                        errorphase = curphase;
1646                else
1647                        errorphase = lastphase;
1648
1649                for (i = 0; i < num_phases; i++) {
1650                        if (errorphase == ahc_phase_table[i].phase)
1651                                break;
1652                }
1653                mesg_out = ahc_phase_table[i].mesg_out;
1654                silent = FALSE;
1655                if (scb != NULL) {
1656                        if (SCB_IS_SILENT(scb))
1657                                silent = TRUE;
1658                        else
1659                                ahc_print_path(ahc, scb);
1660                        scb->flags |= SCB_TRANSMISSION_ERROR;
1661                } else
1662                        printf("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1663                               SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1664                scsirate = ahc_inb(ahc, SCSIRATE);
1665                if (silent == FALSE) {
1666                        printf("parity error detected %s. "
1667                               "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1668                               ahc_phase_table[i].phasemsg,
1669                               ahc_inw(ahc, SEQADDR0),
1670                               scsirate);
1671                        if ((ahc->features & AHC_DT) != 0) {
1672                                if ((sstat2 & CRCVALERR) != 0)
1673                                        printf("\tCRC Value Mismatch\n");
1674                                if ((sstat2 & CRCENDERR) != 0)
1675                                        printf("\tNo terminal CRC packet "
1676                                               "recevied\n");
1677                                if ((sstat2 & CRCREQERR) != 0)
1678                                        printf("\tIllegal CRC packet "
1679                                               "request\n");
1680                                if ((sstat2 & DUAL_EDGE_ERR) != 0)
1681                                        printf("\tUnexpected %sDT Data Phase\n",
1682                                               (scsirate & SINGLE_EDGE)
1683                                             ? "" : "non-");
1684                        }
1685                }
1686
1687                if ((ahc->features & AHC_DT) != 0
1688                 && (sstat2 & DUAL_EDGE_ERR) != 0) {
1689                        /*
1690                         * This error applies regardless of
1691                         * data direction, so ignore the value
1692                         * in the phase table.
1693                         */
1694                        mesg_out = MSG_INITIATOR_DET_ERR;
1695                }
1696
1697                /*
1698                 * We've set the hardware to assert ATN if we   
1699                 * get a parity error on "in" phases, so all we  
1700                 * need to do is stuff the message buffer with
1701                 * the appropriate message.  "In" phases have set
1702                 * mesg_out to something other than MSG_NOP.
1703                 */
1704                if (mesg_out != MSG_NOOP) {
1705                        if (ahc->msg_type != MSG_TYPE_NONE)
1706                                ahc->send_msg_perror = TRUE;
1707                        else
1708                                ahc_outb(ahc, MSG_OUT, mesg_out);
1709                }
1710                /*
1711                 * Force a renegotiation with this target just in
1712                 * case we are out of sync for some external reason
1713                 * unknown (or unreported) by the target.
1714                 */
1715                ahc_fetch_devinfo(ahc, &devinfo);
1716                ahc_force_renegotiation(ahc, &devinfo);
1717
1718                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1719                ahc_unpause(ahc);
1720        } else if ((status & SELTO) != 0) {
1721                u_int        scbptr;
1722
1723                /* Stop the selection */
1724                ahc_outb(ahc, SCSISEQ, 0);
1725
1726                /* No more pending messages */
1727                ahc_clear_msg_state(ahc);
1728
1729                /* Clear interrupt state */
1730                ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1731                ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1732
1733                /*
1734                 * Although the driver does not care about the
1735                 * 'Selection in Progress' status bit, the busy
1736                 * LED does.  SELINGO is only cleared by a sucessfull
1737                 * selection, so we must manually clear it to insure
1738                 * the LED turns off just incase no future successful
1739                 * selections occur (e.g. no devices on the bus).
1740                 */
1741                ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1742
1743                scbptr = ahc_inb(ahc, WAITING_SCBH);
1744                ahc_outb(ahc, SCBPTR, scbptr);
1745                scb_index = ahc_inb(ahc, SCB_TAG);
1746
1747                scb = ahc_lookup_scb(ahc, scb_index);
1748                if (scb == NULL) {
1749                        printf("%s: ahc_intr - referenced scb not "
1750                               "valid during SELTO scb(%d, %d)\n",
1751                               ahc_name(ahc), scbptr, scb_index);
1752                        ahc_dump_card_state(ahc);
1753                } else {
1754                        struct ahc_devinfo devinfo;
1755#ifdef AHC_DEBUG
1756                        if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1757                                ahc_print_path(ahc, scb);
1758                                printf("Saw Selection Timeout for SCB 0x%x\n",
1759                                       scb_index);
1760                        }
1761#endif
1762                        ahc_scb_devinfo(ahc, &devinfo, scb);
1763                        ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1764                        ahc_freeze_devq(ahc, scb);
1765
1766                        /*
1767                         * Cancel any pending transactions on the device
1768                         * now that it seems to be missing.  This will
1769                         * also revert us to async/narrow transfers until
1770                         * we can renegotiate with the device.
1771                         */
1772                        ahc_handle_devreset(ahc, &devinfo,
1773                                            CAM_SEL_TIMEOUT,
1774                                            "Selection Timeout",
1775                                            /*verbose_level*/1);
1776                }
1777                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1778                ahc_restart(ahc);
1779        } else if ((status & BUSFREE) != 0
1780                && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1781                struct        ahc_devinfo devinfo;
1782                u_int        lastphase;
1783                u_int        saved_scsiid;
1784                u_int        saved_lun;
1785                u_int        target;
1786                u_int        initiator_role_id;
1787                char        channel;
1788                int        printerror;
1789
1790                /*
1791                 * Clear our selection hardware as soon as possible.
1792                 * We may have an entry in the waiting Q for this target,
1793                 * that is affected by this busfree and we don't want to
1794                 * go about selecting the target while we handle the event.
1795                 */
1796                ahc_outb(ahc, SCSISEQ,
1797                         ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1798
1799                /*
1800                 * Disable busfree interrupts and clear the busfree
1801                 * interrupt status.  We do this here so that several
1802                 * bus transactions occur prior to clearing the SCSIINT
1803                 * latch.  It can take a bit for the clearing to take effect.
1804                 */
1805                ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1806                ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1807
1808                /*
1809                 * Look at what phase we were last in.
1810                 * If its message out, chances are pretty good
1811                 * that the busfree was in response to one of
1812                 * our abort requests.
1813                 */
1814                lastphase = ahc_inb(ahc, LASTPHASE);
1815                saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1816                saved_lun = ahc_inb(ahc, SAVED_LUN);
1817                target = SCSIID_TARGET(ahc, saved_scsiid);
1818                initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1819                channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1820                ahc_compile_devinfo(&devinfo, initiator_role_id,
1821                                    target, saved_lun, channel, ROLE_INITIATOR);
1822                printerror = 1;
1823
1824                if (lastphase == P_MESGOUT) {
1825                        u_int tag;
1826
1827                        tag = SCB_LIST_NULL;
1828                        if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1829                         || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1830                                if (ahc->msgout_buf[ahc->msgout_index - 1]
1831                                 == MSG_ABORT_TAG)
1832                                        tag = scb->hscb->tag;
1833                                ahc_print_path(ahc, scb);
1834                                printf("SCB %d - Abort%s Completed.\n",
1835                                       scb->hscb->tag, tag == SCB_LIST_NULL ?
1836                                       "" : " Tag");
1837                                ahc_abort_scbs(ahc, target, channel,
1838                                               saved_lun, tag,
1839                                               ROLE_INITIATOR,
1840                                               CAM_REQ_ABORTED);
1841                                printerror = 0;
1842                        } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1843                                                MSG_BUS_DEV_RESET, TRUE)) {
1844#ifdef __FreeBSD__
1845                                /*
1846                                 * Don't mark the user's request for this BDR
1847                                 * as completing with CAM_BDR_SENT.  CAM3
1848                                 * specifies CAM_REQ_CMP.
1849                                 */
1850                                if (scb != NULL
1851                                 && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1852                                 && ahc_match_scb(ahc, scb, target, channel,
1853                                                  CAM_LUN_WILDCARD,
1854                                                  SCB_LIST_NULL,
1855                                                  ROLE_INITIATOR)) {
1856                                        ahc_set_transaction_status(scb, CAM_REQ_CMP);
1857                                }
1858#endif
1859                                ahc_compile_devinfo(&devinfo,
1860                                                    initiator_role_id,
1861                                                    target,
1862                                                    CAM_LUN_WILDCARD,
1863                                                    channel,
1864                                                    ROLE_INITIATOR);
1865                                ahc_handle_devreset(ahc, &devinfo,
1866                                                    CAM_BDR_SENT,
1867                                                    "Bus Device Reset",
1868                                                    /*verbose_level*/0);
1869                                printerror = 0;
1870                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1871                                                MSG_EXT_PPR, FALSE)) {
1872                                struct ahc_initiator_tinfo *tinfo;
1873                                struct ahc_tmode_tstate *tstate;
1874
1875                                /*
1876                                 * PPR Rejected.  Try non-ppr negotiation
1877                                 * and retry command.
1878                                 */
1879                                tinfo = ahc_fetch_transinfo(ahc,
1880                                                            devinfo.channel,
1881                                                            devinfo.our_scsiid,
1882                                                            devinfo.target,
1883                                                            &tstate);
1884                                tinfo->curr.transport_version = 2;
1885                                tinfo->goal.transport_version = 2;
1886                                tinfo->goal.ppr_options = 0;
1887                                ahc_qinfifo_requeue_tail(ahc, scb);
1888                                printerror = 0;
1889                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1890                                                MSG_EXT_WDTR, FALSE)) {
1891                                /*
1892                                 * Negotiation Rejected.  Go-narrow and
1893                                 * retry command.
1894                                 */
1895                                ahc_set_width(ahc, &devinfo,
1896                                              MSG_EXT_WDTR_BUS_8_BIT,
1897                                              AHC_TRANS_CUR|AHC_TRANS_GOAL,
1898                                              /*paused*/TRUE);
1899                                ahc_qinfifo_requeue_tail(ahc, scb);
1900                                printerror = 0;
1901                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1902                                                MSG_EXT_SDTR, FALSE)) {
1903                                /*
1904                                 * Negotiation Rejected.  Go-async and
1905                                 * retry command.
1906                                 */
1907                                ahc_set_syncrate(ahc, &devinfo,
1908                                                /*syncrate*/NULL,
1909                                                /*period*/0, /*offset*/0,
1910                                                /*ppr_options*/0,
1911                                                AHC_TRANS_CUR|AHC_TRANS_GOAL,
1912                                                /*paused*/TRUE);
1913                                ahc_qinfifo_requeue_tail(ahc, scb);
1914                                printerror = 0;
1915                        }
1916                }
1917                if (printerror != 0) {
1918                        u_int i;
1919
1920                        if (scb != NULL) {
1921                                u_int tag;
1922
1923                                if ((scb->hscb->control & TAG_ENB) != 0)
1924                                        tag = scb->hscb->tag;
1925                                else
1926                                        tag = SCB_LIST_NULL;
1927                                ahc_print_path(ahc, scb);
1928                                ahc_abort_scbs(ahc, target, channel,
1929                                               SCB_GET_LUN(scb), tag,
1930                                               ROLE_INITIATOR,
1931                                               CAM_UNEXP_BUSFREE);
1932                        } else {
1933                                /*
1934                                 * We had not fully identified this connection,
1935                                 * so we cannot abort anything.
1936                                 */
1937                                printf("%s: ", ahc_name(ahc));
1938                        }
1939                        for (i = 0; i < num_phases; i++) {
1940                                if (lastphase == ahc_phase_table[i].phase)
1941                                        break;
1942                        }
1943                        if (lastphase != P_BUSFREE) {
1944                                /*
1945                                 * Renegotiate with this device at the
1946                                 * next oportunity just in case this busfree
1947                                 * is due to a negotiation mismatch with the
1948                                 * device.
1949                                 */
1950                                ahc_force_renegotiation(ahc, &devinfo);
1951                        }
1952                        printf("Unexpected busfree %s\n"
1953                               "SEQADDR == 0x%x\n",
1954                               ahc_phase_table[i].phasemsg,
1955                               ahc_inb(ahc, SEQADDR0)
1956                                | (ahc_inb(ahc, SEQADDR1) << 8));
1957                }
1958                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1959                ahc_restart(ahc);
1960        } else {
1961                printf("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1962                       ahc_name(ahc), status);
1963                ahc_outb(ahc, CLRINT, CLRSCSIINT);
1964        }
1965}
1966
1967/*
1968 * Force renegotiation to occur the next time we initiate
1969 * a command to the current device.
1970 */
1971static void
1972ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1973{
1974        struct        ahc_initiator_tinfo *targ_info;
1975        struct        ahc_tmode_tstate *tstate;
1976
1977        targ_info = ahc_fetch_transinfo(ahc,
1978                                        devinfo->channel,
1979                                        devinfo->our_scsiid,
1980                                        devinfo->target,
1981                                        &tstate);
1982        ahc_update_neg_request(ahc, devinfo, tstate,
1983                               targ_info, AHC_NEG_IF_NON_ASYNC);
1984}
1985
1986#define AHC_MAX_STEPS 2000
1987static void
1988ahc_clear_critical_section(struct ahc_softc *ahc)
1989{
1990        int        stepping;
1991        int        steps;
1992        u_int        simode0;
1993        u_int        simode1;
1994
1995        if (ahc->num_critical_sections == 0)
1996                return;
1997
1998        stepping = FALSE;
1999        steps = 0;
2000        simode0 = 0;
2001        simode1 = 0;
2002        for (;;) {
2003                struct        cs *cs;
2004                u_int        seqaddr;
2005                u_int        i;
2006
2007                seqaddr = ahc_inb(ahc, SEQADDR0)
2008                        | (ahc_inb(ahc, SEQADDR1) << 8);
2009
2010                /*
2011                 * Seqaddr represents the next instruction to execute, 
2012                 * so we are really executing the instruction just
2013                 * before it.
2014                 */
2015                if (seqaddr != 0)
2016                        seqaddr -= 1;
2017                cs = ahc->critical_sections;
2018                for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
2019                        
2020                        if (cs->begin < seqaddr && cs->end >= seqaddr)
2021                                break;
2022                }
2023
2024                if (i == ahc->num_critical_sections)
2025                        break;
2026
2027                if (steps > AHC_MAX_STEPS) {
2028                        printf("%s: Infinite loop in critical section\n",
2029                               ahc_name(ahc));
2030                        ahc_dump_card_state(ahc);
2031                        panic("critical section loop");
2032                }
2033
2034                steps++;
2035                if (stepping == FALSE) {
2036
2037                        /*
2038                         * Disable all interrupt sources so that the
2039                         * sequencer will not be stuck by a pausing
2040                         * interrupt condition while we attempt to
2041                         * leave a critical section.
2042                         */
2043                        simode0 = ahc_inb(ahc, SIMODE0);
2044                        ahc_outb(ahc, SIMODE0, 0);
2045                        simode1 = ahc_inb(ahc, SIMODE1);
2046                        if ((ahc->features & AHC_DT) != 0)
2047                                /*
2048                                 * On DT class controllers, we
2049                                 * use the enhanced busfree logic.
2050                                 * Unfortunately we cannot re-enable
2051                                 * busfree detection within the
2052                                 * current connection, so we must
2053                                 * leave it on while single stepping.
2054                                 */
2055                                ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
2056                        else
2057                                ahc_outb(ahc, SIMODE1, 0);
2058                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
2059                        ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
2060                        stepping = TRUE;
2061                }
2062                if ((ahc->features & AHC_DT) != 0) {
2063                        ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
2064                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
2065                }
2066                ahc_outb(ahc, HCNTRL, ahc->unpause);
2067                while (!ahc_is_paused(ahc))
2068                        ahc_delay(200);
2069        }
2070        if (stepping) {
2071                ahc_outb(ahc, SIMODE0, simode0);
2072                ahc_outb(ahc, SIMODE1, simode1);
2073                ahc_outb(ahc, SEQCTL, ahc->seqctl);
2074        }
2075}
2076
2077/*
2078 * Clear any pending interrupt status.
2079 */
2080static void
2081ahc_clear_intstat(struct ahc_softc *ahc)
2082{
2083        /* Clear any interrupt conditions this may have caused */
2084        ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2085                                |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
2086                                CLRREQINIT);
2087        ahc_flush_device_writes(ahc);
2088        ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
2089         ahc_flush_device_writes(ahc);
2090        ahc_outb(ahc, CLRINT, CLRSCSIINT);
2091        ahc_flush_device_writes(ahc);
2092}
2093
2094/**************************** Debugging Routines ******************************/
2095#ifdef AHC_DEBUG
2096uint32_t ahc_debug = AHC_DEBUG_OPTS;
2097#endif
2098
2099#if 0 /* unused */
2100static void
2101ahc_print_scb(struct scb *scb)
2102{
2103        int i;
2104
2105        struct hardware_scb *hscb = scb->hscb;
2106
2107        printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2108               (void *)scb,
2109               hscb->control,
2110               hscb->scsiid,
2111               hscb->lun,
2112               hscb->cdb_len);
2113        printf("Shared Data: ");
2114        for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
2115                printf("%#02x", hscb->shared_data.cdb[i]);
2116        printf("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
2117                ahc_le32toh(hscb->dataptr),
2118                ahc_le32toh(hscb->datacnt),
2119                ahc_le32toh(hscb->sgptr),
2120                hscb->tag);
2121        if (scb->sg_count > 0) {
2122                for (i = 0; i < scb->sg_count; i++) {
2123                        printf("sg[%d] - Addr 0x%x%x : Length %d\n",
2124                               i,
2125                               (ahc_le32toh(scb->sg_list[i].len) >> 24
2126                                & SG_HIGH_ADDR_BITS),
2127                               ahc_le32toh(scb->sg_list[i].addr),
2128                               ahc_le32toh(scb->sg_list[i].len));
2129                }
2130        }
2131}
2132#endif
2133
2134/************************* Transfer Negotiation *******************************/
2135/*
2136 * Allocate per target mode instance (ID we respond to as a target)
2137 * transfer negotiation data structures.
2138 */
2139static struct ahc_tmode_tstate *
2140ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
2141{
2142        struct ahc_tmode_tstate *master_tstate;
2143        struct ahc_tmode_tstate *tstate;
2144        int i;
2145
2146        master_tstate = ahc->enabled_targets[ahc->our_id];
2147        if (channel == 'B') {
2148                scsi_id += 8;
2149                master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
2150        }
2151        if (ahc->enabled_targets[scsi_id] != NULL
2152         && ahc->enabled_targets[scsi_id] != master_tstate)
2153                panic("%s: ahc_alloc_tstate - Target already allocated",
2154                      ahc_name(ahc));
2155        tstate = (struct ahc_tmode_tstate*)malloc(sizeof(*tstate),
2156                                                   M_DEVBUF, M_NOWAIT);
2157        if (tstate == NULL)
2158                return (NULL);
2159
2160        /*
2161         * If we have allocated a master tstate, copy user settings from
2162         * the master tstate (taken from SRAM or the EEPROM) for this
2163         * channel, but reset our current and goal settings to async/narrow
2164         * until an initiator talks to us.
2165         */
2166        if (master_tstate != NULL) {
2167                memcpy(tstate, master_tstate, sizeof(*tstate));
2168                memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2169                tstate->ultraenb = 0;
2170                for (i = 0; i < AHC_NUM_TARGETS; i++) {
2171                        memset(&tstate->transinfo[i].curr, 0,
2172                              sizeof(tstate->transinfo[i].curr));
2173                        memset(&tstate->transinfo[i].goal, 0,
2174                              sizeof(tstate->transinfo[i].goal));
2175                }
2176        } else
2177                memset(tstate, 0, sizeof(*tstate));
2178        ahc->enabled_targets[scsi_id] = tstate;
2179        return (tstate);
2180}
2181
2182#ifdef AHC_TARGET_MODE
2183/*
2184 * Free per target mode instance (ID we respond to as a target)
2185 * transfer negotiation data structures.
2186 */
2187static void
2188ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
2189{
2190        struct ahc_tmode_tstate *tstate;
2191
2192        /*
2193         * Don't clean up our "master" tstate.
2194         * It has our default user settings.
2195         */
2196        if (((channel == 'B' && scsi_id == ahc->our_id_b)
2197          || (channel == 'A' && scsi_id == ahc->our_id))
2198         && force == FALSE)
2199                return;
2200
2201        if (channel == 'B')
2202                scsi_id += 8;
2203        tstate = ahc->enabled_targets[scsi_id];
2204        if (tstate != NULL)
2205                free(tstate, M_DEVBUF);
2206        ahc->enabled_targets[scsi_id] = NULL;
2207}
2208#endif
2209
2210/*
2211 * Called when we have an active connection to a target on the bus,
2212 * this function finds the nearest syncrate to the input period limited
2213 * by the capabilities of the bus connectivity of and sync settings for
2214 * the target.
2215 */
2216const struct ahc_syncrate *
2217ahc_devlimited_syncrate(struct ahc_softc *ahc,
2218                        struct ahc_initiator_tinfo *tinfo,
2219                        u_int *period, u_int *ppr_options, role_t role)
2220{
2221        struct        ahc_transinfo *transinfo;
2222        u_int        maxsync;
2223
2224        if ((ahc->features & AHC_ULTRA2) != 0) {
2225                if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
2226                 && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
2227                        maxsync = AHC_SYNCRATE_DT;
2228                } else {
2229                        maxsync = AHC_SYNCRATE_ULTRA;
2230                        /* Can't do DT on an SE bus */
2231                        *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2232                }
2233        } else if ((ahc->features & AHC_ULTRA) != 0) {
2234                maxsync = AHC_SYNCRATE_ULTRA;
2235        } else {
2236                maxsync = AHC_SYNCRATE_FAST;
2237        }
2238        /*
2239         * Never allow a value higher than our current goal
2240         * period otherwise we may allow a target initiated
2241         * negotiation to go above the limit as set by the
2242         * user.  In the case of an initiator initiated
2243         * sync negotiation, we limit based on the user
2244         * setting.  This allows the system to still accept
2245         * incoming negotiations even if target initiated
2246         * negotiation is not performed.
2247         */
2248        if (role == ROLE_TARGET)
2249                transinfo = &tinfo->user;
2250        else 
2251                transinfo = &tinfo->goal;
2252        *ppr_options &= transinfo->ppr_options;
2253        if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2254                maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
2255                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2256        }
2257        if (transinfo->period == 0) {
2258                *period = 0;
2259                *ppr_options = 0;
2260                return (NULL);
2261        }
2262        *period = max(*period, (u_int)transinfo->period);
2263        return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
2264}
2265
2266/*
2267 * Look up the valid period to SCSIRATE conversion in our table.
2268 * Return the period and offset that should be sent to the target
2269 * if this was the beginning of an SDTR.
2270 */
2271const struct ahc_syncrate *
2272ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
2273                  u_int *ppr_options, u_int maxsync)
2274{
2275        const struct ahc_syncrate *syncrate;
2276
2277        if ((ahc->features & AHC_DT) == 0)
2278                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2279
2280        /* Skip all DT only entries if DT is not available */
2281        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2282         && maxsync < AHC_SYNCRATE_ULTRA2)
2283                maxsync = AHC_SYNCRATE_ULTRA2;
2284
2285        /* Now set the maxsync based on the card capabilities
2286         * DT is already done above */
2287        if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2288            && maxsync < AHC_SYNCRATE_ULTRA)
2289                maxsync = AHC_SYNCRATE_ULTRA;
2290        if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2291            && maxsync < AHC_SYNCRATE_FAST)
2292                maxsync = AHC_SYNCRATE_FAST;
2293
2294        for (syncrate = &ahc_syncrates[maxsync];
2295             syncrate->rate != NULL;
2296             syncrate++) {
2297
2298                /*
2299                 * The Ultra2 table doesn't go as low
2300                 * as for the Fast/Ultra cards.
2301                 */
2302                if ((ahc->features & AHC_ULTRA2) != 0
2303                 && (syncrate->sxfr_u2 == 0))
2304                        break;
2305
2306                if (*period <= syncrate->period) {
2307                        /*
2308                         * When responding to a target that requests
2309                         * sync, the requested rate may fall between
2310                         * two rates that we can output, but still be
2311                         * a rate that we can receive.  Because of this,
2312                         * we want to respond to the target with
2313                         * the same rate that it sent to us even
2314                         * if the period we use to send data to it
2315                         * is lower.  Only lower the response period
2316                         * if we must.
2317                         */
2318                        if (syncrate == &ahc_syncrates[maxsync])
2319                                *period = syncrate->period;
2320
2321                        /*
2322                         * At some speeds, we only support
2323                         * ST transfers.
2324                         */
2325                         if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
2326                                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2327                        break;
2328                }
2329        }
2330
2331        if ((*period == 0)
2332         || (syncrate->rate == NULL)
2333         || ((ahc->features & AHC_ULTRA2) != 0
2334          && (syncrate->sxfr_u2 == 0))) {
2335                /* Use asynchronous transfers. */
2336                *period = 0;
2337                syncrate = NULL;
2338                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2339        }
2340        return (syncrate);
2341}
2342
2343/*
2344 * Convert from an entry in our syncrate table to the SCSI equivalent
2345 * sync "period" factor.
2346 */
2347u_int
2348ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
2349{
2350        const struct ahc_syncrate *syncrate;
2351
2352        if ((ahc->features & AHC_ULTRA2) != 0)
2353                scsirate &= SXFR_ULTRA2;
2354        else
2355                scsirate &= SXFR;
2356
2357        /* now set maxsync based on card capabilities */
2358        if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
2359                maxsync = AHC_SYNCRATE_ULTRA2;
2360        if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2361            && maxsync < AHC_SYNCRATE_ULTRA)
2362                maxsync = AHC_SYNCRATE_ULTRA;
2363        if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2364            && maxsync < AHC_SYNCRATE_FAST)
2365                maxsync = AHC_SYNCRATE_FAST;
2366
2367
2368        syncrate = &ahc_syncrates[maxsync];
2369        while (syncrate->rate != NULL) {
2370
2371                if ((ahc->features & AHC_ULTRA2) != 0) {
2372                        if (syncrate->sxfr_u2 == 0)
2373                                break;
2374                        else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
2375                                return (syncrate->period);
2376                } else if (scsirate == (syncrate->sxfr & SXFR)) {
2377                                return (syncrate->period);
2378                }
2379                syncrate++;
2380        }
2381        return (0); /* async */
2382}
2383
2384/*
2385 * Truncate the given synchronous offset to a value the
2386 * current adapter type and syncrate are capable of.
2387 */
2388static void
2389ahc_validate_offset(struct ahc_softc *ahc,
2390                    struct ahc_initiator_tinfo *tinfo,
2391                    const struct ahc_syncrate *syncrate,
2392                    u_int *offset, int wide, role_t role)
2393{
2394        u_int maxoffset;
2395
2396        /* Limit offset to what we can do */
2397        if (syncrate == NULL) {
2398                maxoffset = 0;
2399        } else if ((ahc->features & AHC_ULTRA2) != 0) {
2400                maxoffset = MAX_OFFSET_ULTRA2;
2401        } else {
2402                if (wide)
2403                        maxoffset = MAX_OFFSET_16BIT;
2404                else
2405                        maxoffset = MAX_OFFSET_8BIT;
2406        }
2407        *offset = min(*offset, maxoffset);
2408        if (tinfo != NULL) {
2409                if (role == ROLE_TARGET)
2410                        *offset = min(*offset, (u_int)tinfo->user.offset);
2411                else
2412                        *offset = min(*offset, (u_int)tinfo->goal.offset);
2413        }
2414}
2415
2416/*
2417 * Truncate the given transfer width parameter to a value the
2418 * current adapter type is capable of.
2419 */
2420static void
2421ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
2422                   u_int *bus_width, role_t role)
2423{
2424        switch (*bus_width) {
2425        default:
2426                if (ahc->features & AHC_WIDE) {
2427                        /* Respond Wide */
2428                        *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2429                        break;
2430                }
2431                /* FALLTHROUGH */
2432        case MSG_EXT_WDTR_BUS_8_BIT:
2433                *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2434                break;
2435        }
2436        if (tinfo != NULL) {
2437                if (role == ROLE_TARGET)
2438                        *bus_width = min((u_int)tinfo->user.width, *bus_width);
2439                else
2440                        *bus_width = min((u_int)tinfo->goal.width, *bus_width);
2441        }
2442}
2443
2444/*
2445 * Update the bitmask of targets for which the controller should
2446 * negotiate with at the next convenient oportunity.  This currently
2447 * means the next time we send the initial identify messages for
2448 * a new transaction.
2449 */
2450int
2451ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2452                       struct ahc_tmode_tstate *tstate,
2453                       struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
2454{
2455        u_int auto_negotiate_orig;
2456
2457        auto_negotiate_orig = tstate->auto_negotiate;
2458        if (neg_type == AHC_NEG_ALWAYS) {
2459                /*
2460                 * Force our "current" settings to be
2461                 * unknown so that unless a bus reset
2462                 * occurs the need to renegotiate is
2463                 * recorded persistently.
2464                 */
2465                if ((ahc->features & AHC_WIDE) != 0)
2466                        tinfo->curr.width = AHC_WIDTH_UNKNOWN;
2467                tinfo->curr.period = AHC_PERIOD_UNKNOWN;
2468                tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
2469        }
2470        if (tinfo->curr.period != tinfo->goal.period
2471         || tinfo->curr.width != tinfo->goal.width
2472         || tinfo->curr.offset != tinfo->goal.offset
2473         || tinfo->curr.ppr_options != tinfo->goal.ppr_options
2474         || (neg_type == AHC_NEG_IF_NON_ASYNC
2475          && (tinfo->goal.offset != 0
2476           || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
2477           || tinfo->goal.ppr_options != 0)))
2478                tstate->auto_negotiate |= devinfo->target_mask;
2479        else
2480                tstate->auto_negotiate &= ~devinfo->target_mask;
2481
2482        return (auto_negotiate_orig != tstate->auto_negotiate);
2483}
2484
2485/*
2486 * Update the user/goal/curr tables of synchronous negotiation
2487 * parameters as well as, in the case of a current or active update,
2488 * any data structures on the host controller.  In the case of an
2489 * active update, the specified target is currently talking to us on
2490 * the bus, so the transfer parameter update must take effect
2491 * immediately.
2492 */
2493void
2494ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2495                 const struct ahc_syncrate *syncrate, u_int period,
2496                 u_int offset, u_int ppr_options, u_int type, int paused)
2497{
2498        struct        ahc_initiator_tinfo *tinfo;
2499        struct        ahc_tmode_tstate *tstate;
2500        u_int        old_period;
2501        u_int        old_offset;
2502        u_int        old_ppr;
2503        int        active;
2504        int        update_needed;
2505
2506        active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2507        update_needed = 0;
2508
2509        if (syncrate == NULL) {
2510                period = 0;
2511                offset = 0;
2512        }
2513
2514        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2515                                    devinfo->target, &tstate);
2516
2517        if ((type & AHC_TRANS_USER) != 0) {
2518                tinfo->user.period = period;
2519                tinfo->user.offset = offset;
2520                tinfo->user.ppr_options = ppr_options;
2521        }
2522
2523        if ((type & AHC_TRANS_GOAL) != 0) {
2524                tinfo->goal.period = period;
2525                tinfo->goal.offset = offset;
2526                tinfo->goal.ppr_options = ppr_options;
2527        }
2528
2529        old_period = tinfo->curr.period;
2530        old_offset = tinfo->curr.offset;
2531        old_ppr           = tinfo->curr.ppr_options;
2532
2533        if ((type & AHC_TRANS_CUR) != 0
2534         && (old_period != period
2535          || old_offset != offset
2536          || old_ppr != ppr_options)) {
2537                u_int        scsirate;
2538
2539                update_needed++;
2540                scsirate = tinfo->scsirate;
2541                if ((ahc->features & AHC_ULTRA2) != 0) {
2542
2543                        scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
2544                        if (syncrate != NULL) {
2545                                scsirate |= syncrate->sxfr_u2;
2546                                if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
2547                                        scsirate |= ENABLE_CRC;
2548                                else
2549                                        scsirate |= SINGLE_EDGE;
2550                        }
2551                } else {
2552
2553                        scsirate &= ~(SXFR|SOFS);
2554                        /*
2555                         * Ensure Ultra mode is set properly for
2556                         * this target.
2557                         */
2558                        tstate->ultraenb &= ~devinfo->target_mask;
2559                        if (syncrate != NULL) {
2560                                if (syncrate->sxfr & ULTRA_SXFR) {
2561                                        tstate->ultraenb |=
2562                                                devinfo->target_mask;
2563                                }
2564                                scsirate |= syncrate->sxfr & SXFR;
2565                                scsirate |= offset & SOFS;
2566                        }
2567                        if (active) {
2568                                u_int sxfrctl0;
2569
2570                                sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
2571                                sxfrctl0 &= ~FAST20;
2572                                if (tstate->ultraenb & devinfo->target_mask)
2573                                        sxfrctl0 |= FAST20;
2574                                ahc_outb(ahc, SXFRCTL0, sxfrctl0);
2575                        }
2576                }
2577                if (active) {
2578                        ahc_outb(ahc, SCSIRATE, scsirate);
2579                        if ((ahc->features & AHC_ULTRA2) != 0)
2580                                ahc_outb(ahc, SCSIOFFSET, offset);
2581                }
2582
2583                tinfo->scsirate = scsirate;
2584                tinfo->curr.period = period;
2585                tinfo->curr.offset = offset;
2586                tinfo->curr.ppr_options = ppr_options;
2587
2588                ahc_send_async(ahc, devinfo->channel, devinfo->target,
2589                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2590                if (bootverbose) {
2591                        if (offset != 0) {
2592                                printf("%s: target %d synchronous at %sMHz%s, "
2593                                       "offset = 0x%x\n", ahc_name(ahc),
2594                                       devinfo->target, syncrate->rate,
2595                                       (ppr_options & MSG_EXT_PPR_DT_REQ)
2596                                       ? " DT" : "", offset);
2597                        } else {
2598                                printf("%s: target %d using "
2599                                       "asynchronous transfers\n",
2600                                       ahc_name(ahc), devinfo->target);
2601                        }
2602                }
2603        }
2604
2605        update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2606                                                tinfo, AHC_NEG_TO_GOAL);
2607
2608        if (update_needed)
2609                ahc_update_pending_scbs(ahc);
2610}
2611
2612/*
2613 * Update the user/goal/curr tables of wide negotiation
2614 * parameters as well as, in the case of a current or active update,
2615 * any data structures on the host controller.  In the case of an
2616 * active update, the specified target is currently talking to us on
2617 * the bus, so the transfer parameter update must take effect
2618 * immediately.
2619 */
2620void
2621ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2622              u_int width, u_int type, int paused)
2623{
2624        struct        ahc_initiator_tinfo *tinfo;
2625        struct        ahc_tmode_tstate *tstate;
2626        u_int        oldwidth;
2627        int        active;
2628        int        update_needed;
2629
2630        active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2631        update_needed = 0;
2632        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2633                                    devinfo->target, &tstate);
2634
2635        if ((type & AHC_TRANS_USER) != 0)
2636                tinfo->user.width = width;
2637
2638        if ((type & AHC_TRANS_GOAL) != 0)
2639                tinfo->goal.width = width;
2640
2641        oldwidth = tinfo->curr.width;
2642        if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2643                u_int        scsirate;
2644
2645                update_needed++;
2646                scsirate =  tinfo->scsirate;
2647                scsirate &= ~WIDEXFER;
2648                if (width == MSG_EXT_WDTR_BUS_16_BIT)
2649                        scsirate |= WIDEXFER;
2650
2651                tinfo->scsirate = scsirate;
2652
2653                if (active)
2654                        ahc_outb(ahc, SCSIRATE, scsirate);
2655
2656                tinfo->curr.width = width;
2657
2658                ahc_send_async(ahc, devinfo->channel, devinfo->target,
2659                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2660                if (bootverbose) {
2661                        printf("%s: target %d using %dbit transfers\n",
2662                               ahc_name(ahc), devinfo->target,
2663                               8 * (0x01 << width));
2664                }
2665        }
2666
2667        update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2668                                                tinfo, AHC_NEG_TO_GOAL);
2669        if (update_needed)
2670                ahc_update_pending_scbs(ahc);
2671}
2672
2673/*
2674 * Update the current state of tagged queuing for a given target.
2675 */
2676static void
2677ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
2678             struct ahc_devinfo *devinfo, ahc_queue_alg alg)
2679{
2680        struct scsi_device *sdev = cmd->device;
2681
2682         ahc_platform_set_tags(ahc, sdev, devinfo, alg);
2683         ahc_send_async(ahc, devinfo->channel, devinfo->target,
2684                        devinfo->lun, AC_TRANSFER_NEG);
2685}
2686
2687/*
2688 * When the transfer settings for a connection change, update any
2689 * in-transit SCBs to contain the new data so the hardware will
2690 * be set correctly during future (re)selections.
2691 */
2692static void
2693ahc_update_pending_scbs(struct ahc_softc *ahc)
2694{
2695        struct        scb *pending_scb;
2696        int        pending_scb_count;
2697        int        i;
2698        int        paused;
2699        u_int        saved_scbptr;
2700
2701        /*
2702         * Traverse the pending SCB list and ensure that all of the
2703         * SCBs there have the proper settings.
2704         */
2705        pending_scb_count = 0;
2706        LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2707                struct ahc_devinfo devinfo;
2708                struct hardware_scb *pending_hscb;
2709                struct ahc_initiator_tinfo *tinfo;
2710                struct ahc_tmode_tstate *tstate;
2711
2712                ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2713                tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2714                                            devinfo.our_scsiid,
2715                                            devinfo.target, &tstate);
2716                pending_hscb = pending_scb->hscb;
2717                pending_hscb->control &= ~ULTRAENB;
2718                if ((tstate->ultraenb & devinfo.target_mask) != 0)
2719                        pending_hscb->control |= ULTRAENB;
2720                pending_hscb->scsirate = tinfo->scsirate;
2721                pending_hscb->scsioffset = tinfo->curr.offset;
2722                if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2723                 && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2724                        pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2725                        pending_hscb->control &= ~MK_MESSAGE;
2726                }
2727                ahc_sync_scb(ahc, pending_scb,
2728                             BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2729                pending_scb_count++;
2730        }
2731
2732        if (pending_scb_count == 0)
2733                return;
2734
2735        if (ahc_is_paused(ahc)) {
2736                paused = 1;
2737        } else {
2738                paused = 0;
2739                ahc_pause(ahc);
2740        }
2741
2742        saved_scbptr = ahc_inb(ahc, SCBPTR);
2743        /* Ensure that the hscbs down on the card match the new information */
2744        for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2745                struct        hardware_scb *pending_hscb;
2746                u_int        control;
2747                u_int        scb_tag;
2748
2749                ahc_outb(ahc, SCBPTR, i);
2750                scb_tag = ahc_inb(ahc, SCB_TAG);
2751                pending_scb = ahc_lookup_scb(ahc, scb_tag);
2752                if (pending_scb == NULL)
2753                        continue;
2754
2755                pending_hscb = pending_scb->hscb;
2756                control = ahc_inb(ahc, SCB_CONTROL);
2757                control &= ~(ULTRAENB|MK_MESSAGE);
2758                control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2759                ahc_outb(ahc, SCB_CONTROL, control);
2760                ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2761                ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2762        }
2763        ahc_outb(ahc, SCBPTR, saved_scbptr);
2764
2765        if (paused == 0)
2766                ahc_unpause(ahc);
2767}
2768
2769/**************************** Pathing Information *****************************/
2770static void
2771ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2772{
2773        u_int        saved_scsiid;
2774        role_t        role;
2775        int        our_id;
2776
2777        if (ahc_inb(ahc, SSTAT0) & TARGET)
2778                role = ROLE_TARGET;
2779        else
2780                role = ROLE_INITIATOR;
2781
2782        if (role == ROLE_TARGET
2783         && (ahc->features & AHC_MULTI_TID) != 0
2784         && (ahc_inb(ahc, SEQ_FLAGS)
2785            & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2786                /* We were selected, so pull our id from TARGIDIN */
2787                our_id = ahc_inb(ahc, TARGIDIN) & OID;
2788        } else if ((ahc->features & AHC_ULTRA2) != 0)
2789                our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2790        else
2791                our_id = ahc_inb(ahc, SCSIID) & OID;
2792
2793        saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2794        ahc_compile_devinfo(devinfo,
2795                            our_id,
2796                            SCSIID_TARGET(ahc, saved_scsiid),
2797                            ahc_inb(ahc, SAVED_LUN),
2798                            SCSIID_CHANNEL(ahc, saved_scsiid),
2799                            role);
2800}
2801
2802static const struct ahc_phase_table_entry*
2803ahc_lookup_phase_entry(int phase)
2804{
2805        const struct ahc_phase_table_entry *entry;
2806        const struct ahc_phase_table_entry *last_entry;
2807
2808        /*
2809         * num_phases doesn't include the default entry which
2810         * will be returned if the phase doesn't match.
2811         */
2812        last_entry = &ahc_phase_table[num_phases];
2813        for (entry = ahc_phase_table; entry < last_entry; entry++) {
2814                if (phase == entry->phase)
2815                        break;
2816        }
2817        return (entry);
2818}
2819
2820void
2821ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2822                    u_int lun, char channel, role_t role)
2823{
2824        devinfo->our_scsiid = our_id;
2825        devinfo->target = target;
2826        devinfo->lun = lun;
2827        devinfo->target_offset = target;
2828        devinfo->channel = channel;
2829        devinfo->role = role;
2830        if (channel == 'B')
2831                devinfo->target_offset += 8;
2832        devinfo->target_mask = (0x01 << devinfo->target_offset);
2833}
2834
2835void
2836ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2837{
2838        printf("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2839               devinfo->target, devinfo->lun);
2840}
2841
2842static void
2843ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2844                struct scb *scb)
2845{
2846        role_t        role;
2847        int        our_id;
2848
2849        our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2850        role = ROLE_INITIATOR;
2851        if ((scb->flags & SCB_TARGET_SCB) != 0)
2852                role = ROLE_TARGET;
2853        ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2854                            SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2855}
2856
2857
2858/************************ Message Phase Processing ****************************/
2859static void
2860ahc_assert_atn(struct ahc_softc *ahc)
2861{
2862        u_int scsisigo;
2863
2864        scsisigo = ATNO;
2865        if ((ahc->features & AHC_DT) == 0)
2866                scsisigo |= ahc_inb(ahc, SCSISIGI);
2867        ahc_outb(ahc, SCSISIGO, scsisigo);
2868}
2869
2870/*
2871 * When an initiator transaction with the MK_MESSAGE flag either reconnects
2872 * or enters the initial message out phase, we are interrupted.  Fill our
2873 * outgoing message buffer with the appropriate message and beging handing
2874 * the message phase(s) manually.
2875 */
2876static void
2877ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2878                           struct scb *scb)
2879{
2880        /*
2881         * To facilitate adding multiple messages together,
2882         * each routine should increment the index and len
2883         * variables instead of setting them explicitly.
2884         */
2885        ahc->msgout_index = 0;
2886        ahc->msgout_len = 0;
2887
2888        if ((scb->flags & SCB_DEVICE_RESET) == 0
2889         && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2890                u_int identify_msg;
2891
2892                identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2893                if ((scb->hscb->control & DISCENB) != 0)
2894                        identify_msg |= MSG_IDENTIFY_DISCFLAG;
2895                ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2896                ahc->msgout_len++;
2897
2898                if ((scb->hscb->control & TAG_ENB) != 0) {
2899                        ahc->msgout_buf[ahc->msgout_index++] =
2900                            scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2901                        ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2902                        ahc->msgout_len += 2;
2903                }
2904        }
2905
2906        if (scb->flags & SCB_DEVICE_RESET) {
2907                ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2908                ahc->msgout_len++;
2909                ahc_print_path(ahc, scb);
2910                printf("Bus Device Reset Message Sent\n");
2911                /*
2912                 * Clear our selection hardware in advance of
2913                 * the busfree.  We may have an entry in the waiting
2914                 * Q for this target, and we don't want to go about
2915                 * selecting while we handle the busfree and blow it
2916                 * away.
2917                 */
2918                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2919        } else if ((scb->flags & SCB_ABORT) != 0) {
2920                if ((scb->hscb->control & TAG_ENB) != 0)
2921                        ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2922                else
2923                        ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2924                ahc->msgout_len++;
2925                ahc_print_path(ahc, scb);
2926                printf("Abort%s Message Sent\n",
2927                       (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2928                /*
2929                 * Clear our selection hardware in advance of
2930                 * the busfree.  We may have an entry in the waiting
2931                 * Q for this target, and we don't want to go about
2932                 * selecting while we handle the busfree and blow it
2933                 * away.
2934                 */
2935                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2936        } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2937                ahc_build_transfer_msg(ahc, devinfo);
2938        } else {
2939                printf("ahc_intr: AWAITING_MSG for an SCB that "
2940                       "does not have a waiting message\n");
2941                printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2942                       devinfo->target_mask);
2943                panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2944                      "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2945                      ahc_inb(ahc, MSG_OUT), scb->flags);
2946        }
2947
2948        /*
2949         * Clear the MK_MESSAGE flag from the SCB so we aren't
2950         * asked to send this message again.
2951         */
2952        ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2953        scb->hscb->control &= ~MK_MESSAGE;
2954        ahc->msgout_index = 0;
2955        ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2956}
2957
2958/*
2959 * Build an appropriate transfer negotiation message for the
2960 * currently active target.
2961 */
2962static void
2963ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2964{
2965        /*
2966         * We need to initiate transfer negotiations.
2967         * If our current and goal settings are identical,
2968         * we want to renegotiate due to a check condition.
2969         */
2970        struct        ahc_initiator_tinfo *tinfo;
2971        struct        ahc_tmode_tstate *tstate;
2972        const struct ahc_syncrate *rate;
2973        int        dowide;
2974        int        dosync;
2975        int        doppr;
2976        u_int        period;
2977        u_int        ppr_options;
2978        u_int        offset;
2979
2980        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2981                                    devinfo->target, &tstate);
2982        /*
2983         * Filter our period based on the current connection.
2984         * If we can't perform DT transfers on this segment (not in LVD
2985         * mode for instance), then our decision to issue a PPR message
2986         * may change.
2987         */
2988        period = tinfo->goal.period;
2989        offset = tinfo->goal.offset;
2990        ppr_options = tinfo->goal.ppr_options;
2991        /* Target initiated PPR is not allowed in the SCSI spec */
2992        if (devinfo->role == ROLE_TARGET)
2993                ppr_options = 0;
2994        rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2995                                       &ppr_options, devinfo->role);
2996        dowide = tinfo->curr.width != tinfo->goal.width;
2997        dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2998        /*
2999         * Only use PPR if we have options that need it, even if the device
3000         * claims to support it.  There might be an expander in the way
3001         * that doesn't.
3002         */
3003        doppr = ppr_options != 0;
3004
3005        if (!dowide && !dosync && !doppr) {
3006                dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
3007                dosync = tinfo->goal.offset != 0;
3008        }
3009
3010        if (!dowide && !dosync && !doppr) {
3011                /*
3012                 * Force async with a WDTR message if we have a wide bus,
3013                 * or just issue an SDTR with a 0 offset.
3014                 */
3015                if ((ahc->features & AHC_WIDE) != 0)
3016                        dowide = 1;
3017                else
3018                        dosync = 1;
3019
3020                if (bootverbose) {
3021                        ahc_print_devinfo(ahc, devinfo);
3022                        printf("Ensuring async\n");
3023                }
3024        }
3025
3026        /* Target initiated PPR is not allowed in the SCSI spec */
3027        if (devinfo->role == ROLE_TARGET)
3028                doppr = 0;
3029
3030        /*
3031         * Both the PPR message and SDTR message require the
3032         * goal syncrate to be limited to what the target device
3033         * is capable of handling (based on whether an LVD->SE
3034         * expander is on the bus), so combine these two cases.
3035         * Regardless, guarantee that if we are using WDTR and SDTR
3036         * messages that WDTR comes first.
3037         */
3038        if (doppr || (dosync && !dowide)) {
3039
3040                offset = tinfo->goal.offset;
3041                ahc_validate_offset(ahc, tinfo, rate, &offset,
3042                                    doppr ? tinfo->goal.width
3043                                          : tinfo->curr.width,
3044                                    devinfo->role);
3045                if (doppr) {
3046                        ahc_construct_ppr(ahc, devinfo, period, offset,
3047                                          tinfo->goal.width, ppr_options);
3048                } else {
3049                        ahc_construct_sdtr(ahc, devinfo, period, offset);
3050                }
3051        } else {
3052                ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
3053        }
3054}
3055
3056/*
3057 * Build a synchronous negotiation message in our message
3058 * buffer based on the input parameters.
3059 */
3060static void
3061ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3062                   u_int period, u_int offset)
3063{
3064        if (offset == 0)
3065                period = AHC_ASYNC_XFER_PERIOD;
3066        ahc->msgout_index += spi_populate_sync_msg(
3067                        ahc->msgout_buf + ahc->msgout_index, period, offset);
3068        ahc->msgout_len += 5;
3069        if (bootverbose) {
3070                printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
3071                       ahc_name(ahc), devinfo->channel, devinfo->target,
3072                       devinfo->lun, period, offset);
3073        }
3074}
3075
3076/*
3077 * Build a wide negotiation message in our message
3078 * buffer based on the input parameters.
3079 */
3080static void
3081ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3082                   u_int bus_width)
3083{
3084        ahc->msgout_index += spi_populate_width_msg(
3085                        ahc->msgout_buf + ahc->msgout_index, bus_width);
3086        ahc->msgout_len += 4;
3087        if (bootverbose) {
3088                printf("(%s:%c:%d:%d): Sending WDTR %x\n",
3089                       ahc_name(ahc), devinfo->channel, devinfo->target,
3090                       devinfo->lun, bus_width);
3091        }
3092}
3093
3094/*
3095 * Build a parallel protocol request message in our message
3096 * buffer based on the input parameters.
3097 */
3098static void
3099ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3100                  u_int period, u_int offset, u_int bus_width,
3101                  u_int ppr_options)
3102{
3103        if (offset == 0)
3104                period = AHC_ASYNC_XFER_PERIOD;
3105        ahc->msgout_index += spi_populate_ppr_msg(
3106                        ahc->msgout_buf + ahc->msgout_index, period, offset,
3107                        bus_width, ppr_options);
3108        ahc->msgout_len += 8;
3109        if (bootverbose) {
3110                printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3111                       "offset %x, ppr_options %x\n", ahc_name(ahc),
3112                       devinfo->channel, devinfo->target, devinfo->lun,
3113                       bus_width, period, offset, ppr_options);
3114        }
3115}
3116
3117/*
3118 * Clear any active message state.
3119 */
3120static void
3121ahc_clear_msg_state(struct ahc_softc *ahc)
3122{
3123        ahc->msgout_len = 0;
3124        ahc->msgin_index = 0;
3125        ahc->msg_type = MSG_TYPE_NONE;
3126        if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
3127                /*
3128                 * The target didn't care to respond to our
3129                 * message request, so clear ATN.
3130                 */
3131                ahc_outb(ahc, CLRSINT1, CLRATNO);
3132        }
3133        ahc_outb(ahc, MSG_OUT, MSG_NOOP);
3134        ahc_outb(ahc, SEQ_FLAGS2,
3135                 ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3136}
3137
3138static void
3139ahc_handle_proto_violation(struct ahc_softc *ahc)
3140{
3141        struct        ahc_devinfo devinfo;
3142        struct        scb *scb;
3143        u_int        scbid;
3144        u_int        seq_flags;
3145        u_int        curphase;
3146        u_int        lastphase;
3147        int        found;
3148
3149        ahc_fetch_devinfo(ahc, &devinfo);
3150        scbid = ahc_inb(ahc, SCB_TAG);
3151        scb = ahc_lookup_scb(ahc, scbid);
3152        seq_flags = ahc_inb(ahc, SEQ_FLAGS);
3153        curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3154        lastphase = ahc_inb(ahc, LASTPHASE);
3155        if ((seq_flags & NOT_IDENTIFIED) != 0) {
3156
3157                /*
3158                 * The reconnecting target either did not send an
3159                 * identify message, or did, but we didn't find an SCB
3160                 * to match.
3161                 */
3162                ahc_print_devinfo(ahc, &devinfo);
3163                printf("Target did not send an IDENTIFY message. "
3164                       "LASTPHASE = 0x%x.\n", lastphase);
3165                scb = NULL;
3166        } else if (scb == NULL) {
3167                /*
3168                 * We don't seem to have an SCB active for this
3169                 * transaction.  Print an error and reset the bus.
3170                 */
3171                ahc_print_devinfo(ahc, &devinfo);
3172                printf("No SCB found during protocol violation\n");
3173                goto proto_violation_reset;
3174        } else {
3175                ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3176                if ((seq_flags & NO_CDB_SENT) != 0) {
3177                        ahc_print_path(ahc, scb);
3178                        printf("No or incomplete CDB sent to device.\n");
3179                } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
3180                        /*
3181                         * The target never bothered to provide status to
3182                         * us prior to completing the command.  Since we don't
3183                         * know the disposition of this command, we must attempt
3184                         * to abort it.  Assert ATN and prepare to send an abort
3185                         * message.
3186                         */
3187                        ahc_print_path(ahc, scb);
3188                        printf("Completed command without status.\n");
3189                } else {
3190                        ahc_print_path(ahc, scb);
3191                        printf("Unknown protocol violation.\n");
3192                        ahc_dump_card_state(ahc);
3193                }
3194        }
3195        if ((lastphase & ~P_DATAIN_DT) == 0
3196         || lastphase == P_COMMAND) {
3197proto_violation_reset:
3198                /*
3199                 * Target either went directly to data/command
3200                 * phase or didn't respond to our ATN.
3201                 * The only safe thing to do is to blow
3202                 * it away with a bus reset.
3203                 */
3204                found = ahc_reset_channel(ahc, 'A', TRUE);
3205                printf("%s: Issued Channel %c Bus Reset. "
3206                       "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
3207        } else {
3208                /*
3209                 * Leave the selection hardware off in case
3210                 * this abort attempt will affect yet to
3211                 * be sent commands.
3212                 */
3213                ahc_outb(ahc, SCSISEQ,
3214                         ahc_inb(ahc, SCSISEQ) & ~ENSELO);
3215                ahc_assert_atn(ahc);
3216                ahc_outb(ahc, MSG_OUT, HOST_MSG);
3217                if (scb == NULL) {
3218                        ahc_print_devinfo(ahc, &devinfo);
3219                        ahc->msgout_buf[0] = MSG_ABORT_TASK;
3220                        ahc->msgout_len = 1;
3221                        ahc->msgout_index = 0;
3222                        ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3223                } else {
3224                        ahc_print_path(ahc, scb);
3225                        scb->flags |= SCB_ABORT;
3226                }
3227                printf("Protocol violation %s.  Attempting to abort.\n",
3228                       ahc_lookup_phase_entry(curphase)->phasemsg);
3229        }
3230}
3231
3232/*
3233 * Manual message loop handler.
3234 */
3235static void
3236ahc_handle_message_phase(struct ahc_softc *ahc)
3237{
3238        struct        ahc_devinfo devinfo;
3239        u_int        bus_phase;
3240        int        end_session;
3241
3242        ahc_fetch_devinfo(ahc, &devinfo);
3243        end_session = FALSE;
3244        bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3245
3246reswitch:
3247        switch (ahc->msg_type) {
3248        case MSG_TYPE_INITIATOR_MSGOUT:
3249        {
3250                int lastbyte;
3251                int phasemis;
3252                int msgdone;
3253
3254                if (ahc->msgout_len == 0)
3255                        panic("HOST_MSG_LOOP interrupt with no active message");
3256
3257#ifdef AHC_DEBUG
3258                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3259                        ahc_print_devinfo(ahc, &devinfo);
3260                        printf("INITIATOR_MSG_OUT");
3261                }
3262#endif
3263                phasemis = bus_phase != P_MESGOUT;
3264                if (phasemis) {
3265#ifdef AHC_DEBUG
3266                        if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3267                                printf(" PHASEMIS %s\n",
3268                                       ahc_lookup_phase_entry(bus_phase)
3269                                                             ->phasemsg);
3270                        }
3271#endif
3272                        if (bus_phase == P_MESGIN) {
3273                                /*
3274                                 * Change gears and see if
3275                                 * this messages is of interest to
3276                                 * us or should be passed back to
3277                                 * the sequencer.
3278                                 */
3279                                ahc_outb(ahc, CLRSINT1, CLRATNO);
3280                                ahc->send_msg_perror = FALSE;
3281                                ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3282                                ahc->msgin_index = 0;
3283                                goto reswitch;
3284                        }
3285                        end_session = TRUE;
3286                        break;
3287                }
3288
3289                if (ahc->send_msg_perror) {
3290                        ahc_outb(ahc, CLRSINT1, CLRATNO);
3291                        ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3292#ifdef AHC_DEBUG
3293                        if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3294                                printf(" byte 0x%x\n", ahc->send_msg_perror);
3295#endif
3296                        ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
3297                        break;
3298                }
3299
3300                msgdone        = ahc->msgout_index == ahc->msgout_len;
3301                if (msgdone) {
3302                        /*
3303                         * The target has requested a retry.
3304                         * Re-assert ATN, reset our message index to
3305                         * 0, and try again.
3306                         */
3307                        ahc->msgout_index = 0;
3308                        ahc_assert_atn(ahc);
3309                }
3310
3311                lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
3312                if (lastbyte) {
3313                        /* Last byte is signified by dropping ATN */
3314                        ahc_outb(ahc, CLRSINT1, CLRATNO);
3315                }
3316
3317                /*
3318                 * Clear our interrupt status and present
3319                 * the next byte on the bus.
3320                 */
3321                ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3322#ifdef AHC_DEBUG
3323                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3324                        printf(" byte 0x%x\n",
3325                               ahc->msgout_buf[ahc->msgout_index]);
3326#endif
3327                ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3328                break;
3329        }
3330        case MSG_TYPE_INITIATOR_MSGIN:
3331        {
3332                int phasemis;
3333                int message_done;
3334
3335#ifdef AHC_DEBUG
3336                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3337                        ahc_print_devinfo(ahc, &devinfo);
3338                        printf("INITIATOR_MSG_IN");
3339                }
3340#endif
3341                phasemis = bus_phase != P_MESGIN;
3342                if (phasemis) {
3343#ifdef AHC_DEBUG
3344                        if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3345                                printf(" PHASEMIS %s\n",
3346                                       ahc_lookup_phase_entry(bus_phase)
3347                                                             ->phasemsg);
3348                        }
3349#endif
3350                        ahc->msgin_index = 0;
3351                        if (bus_phase == P_MESGOUT
3352                         && (ahc->send_msg_perror == TRUE
3353                          || (ahc->msgout_len != 0
3354                           && ahc->msgout_index == 0))) {
3355                                ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3356                                goto reswitch;
3357                        }
3358                        end_session = TRUE;
3359                        break;
3360                }
3361
3362                /* Pull the byte in without acking it */
3363                ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
3364#ifdef AHC_DEBUG
3365                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3366                        printf(" byte 0x%x\n",
3367                               ahc->msgin_buf[ahc->msgin_index]);
3368#endif
3369
3370                message_done = ahc_parse_msg(ahc, &devinfo);
3371
3372                if (message_done) {
3373                        /*
3374                         * Clear our incoming message buffer in case there
3375                         * is another message following this one.
3376                         */
3377                        ahc->msgin_index = 0;
3378
3379                        /*
3380                         * If this message illicited a response,
3381                         * assert ATN so the target takes us to the
3382                         * message out phase.
3383                         */
3384                        if (ahc->msgout_len != 0) {
3385#ifdef AHC_DEBUG
3386                                if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3387                                        ahc_print_devinfo(ahc, &devinfo);
3388                                        printf("Asserting ATN for response\n");
3389                                }
3390#endif
3391                                ahc_assert_atn(ahc);
3392                        }
3393                } else 
3394                        ahc->msgin_index++;
3395
3396                if (message_done == MSGLOOP_TERMINATED) {
3397                        end_session = TRUE;
3398                } else {
3399                        /* Ack the byte */
3400                        ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3401                        ahc_inb(ahc, SCSIDATL);
3402                }
3403                break;
3404        }
3405        case MSG_TYPE_TARGET_MSGIN:
3406        {
3407                int msgdone;
3408                int msgout_request;
3409
3410                if (ahc->msgout_len == 0)
3411                        panic("Target MSGIN with no active message");
3412
3413                /*
3414                 * If we interrupted a mesgout session, the initiator
3415                 * will not know this until our first REQ.  So, we
3416                 * only honor mesgout requests after we've sent our
3417                 * first byte.
3418                 */
3419                if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
3420                 && ahc->msgout_index > 0)
3421                        msgout_request = TRUE;
3422                else
3423                        msgout_request = FALSE;
3424
3425                if (msgout_request) {
3426
3427                        /*
3428                         * Change gears and see if
3429                         * this messages is of interest to
3430                         * us or should be passed back to
3431                         * the sequencer.
3432                         */
3433                        ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
3434                        ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
3435                        ahc->msgin_index = 0;
3436                        /* Dummy read to REQ for first byte */
3437                        ahc_inb(ahc, SCSIDATL);
3438                        ahc_outb(ahc, SXFRCTL0,
3439                                 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3440                        break;
3441                }
3442
3443                msgdone = ahc->msgout_index == ahc->msgout_len;
3444                if (msgdone) {
3445                        ahc_outb(ahc, SXFRCTL0,
3446                                 ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3447                        end_session = TRUE;
3448                        break;
3449                }
3450
3451                /*
3452                 * Present the next byte on the bus.
3453                 */
3454                ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3455                ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3456                break;
3457        }
3458        case MSG_TYPE_TARGET_MSGOUT:
3459        {
3460                int lastbyte;
3461                int msgdone;
3462
3463                /*
3464                 * The initiator signals that this is
3465                 * the last byte by dropping ATN.
3466                 */
3467                lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
3468
3469                /*
3470                 * Read the latched byte, but turn off SPIOEN first
3471                 * so that we don't inadvertently cause a REQ for the
3472                 * next byte.
3473                 */
3474                ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3475                ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
3476                msgdone = ahc_parse_msg(ahc, &devinfo);
3477                if (msgdone == MSGLOOP_TERMINATED) {
3478                        /*
3479                         * The message is *really* done in that it caused
3480                         * us to go to bus free.  The sequencer has already
3481                         * been reset at this point, so pull the ejection
3482                         * handle.
3483                         */
3484                        return;
3485                }
3486                
3487                ahc->msgin_index++;
3488
3489                /*
3490                 * XXX Read spec about initiator dropping ATN too soon
3491                 *     and use msgdone to detect it.
3492                 */
3493                if (msgdone == MSGLOOP_MSGCOMPLETE) {
3494                        ahc->msgin_index = 0;
3495
3496                        /*
3497                         * If this message illicited a response, transition
3498                         * to the Message in phase and send it.
3499                         */
3500                        if (ahc->msgout_len != 0) {
3501                                ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
3502                                ahc_outb(ahc, SXFRCTL0,
3503                                         ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3504                                ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3505                                ahc->msgin_index = 0;
3506                                break;
3507                        }
3508                }
3509
3510                if (lastbyte)
3511                        end_session = TRUE;
3512                else {
3513                        /* Ask for the next byte. */
3514                        ahc_outb(ahc, SXFRCTL0,
3515                                 ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3516                }
3517
3518                break;
3519        }
3520        default:
3521                panic("Unknown REQINIT message type");
3522        }
3523
3524        if (end_session) {
3525                ahc_clear_msg_state(ahc);
3526                ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
3527        } else
3528                ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
3529}
3530
3531/*
3532 * See if we sent a particular extended message to the target.
3533 * If "full" is true, return true only if the target saw the full
3534 * message.  If "full" is false, return true if the target saw at
3535 * least the first byte of the message.
3536 */
3537static int
3538ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
3539{
3540        int found;
3541        u_int index;
3542
3543        found = FALSE;
3544        index = 0;
3545
3546        while (index < ahc->msgout_len) {
3547                if (ahc->msgout_buf[index] == MSG_EXTENDED) {
3548                        u_int end_index;
3549
3550                        end_index = index + 1 + ahc->msgout_buf[index + 1];
3551                        if (ahc->msgout_buf[index+2] == msgval
3552                         && type == AHCMSG_EXT) {
3553
3554                                if (full) {
3555                                        if (ahc->msgout_index > end_index)
3556                                                found = TRUE;
3557                                } else if (ahc->msgout_index > index)
3558                                        found = TRUE;
3559                        }
3560                        index = end_index;
3561                } else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
3562                        && ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
3563
3564                        /* Skip tag type and tag id or residue param*/
3565                        index += 2;
3566                } else {
3567                        /* Single byte message */
3568                        if (type == AHCMSG_1B
3569                         && ahc->msgout_buf[index] == msgval
3570                         && ahc->msgout_index > index)
3571                                found = TRUE;
3572                        index++;
3573                }
3574
3575                if (found)
3576                        break;
3577        }
3578        return (found);
3579}
3580
3581/*
3582 * Wait for a complete incoming message, parse it, and respond accordingly.
3583 */
3584static int
3585ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3586{
3587        struct        ahc_initiator_tinfo *tinfo;
3588        struct        ahc_tmode_tstate *tstate;
3589        int        reject;
3590        int        done;
3591        int        response;
3592        u_int        targ_scsirate;
3593
3594        done = MSGLOOP_IN_PROG;
3595        response = FALSE;
3596        reject = FALSE;
3597        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
3598                                    devinfo->target, &tstate);
3599        targ_scsirate = tinfo->scsirate;
3600
3601        /*
3602         * Parse as much of the message as is available,
3603         * rejecting it if we don't support it.  When
3604         * the entire message is available and has been
3605         * handled, return MSGLOOP_MSGCOMPLETE, indicating
3606         * that we have parsed an entire message.
3607         *
3608         * In the case of extended messages, we accept the length
3609         * byte outright and perform more checking once we know the
3610         * extended message type.
3611         */
3612        switch (ahc->msgin_buf[0]) {
3613        case MSG_DISCONNECT:
3614        case MSG_SAVEDATAPOINTER:
3615        case MSG_CMDCOMPLETE:
3616        case MSG_RESTOREPOINTERS:
3617        case MSG_IGN_WIDE_RESIDUE:
3618                /*
3619                 * End our message loop as these are messages
3620                 * the sequencer handles on its own.
3621                 */
3622                done = MSGLOOP_TERMINATED;
3623                break;
3624        case MSG_MESSAGE_REJECT:
3625                response = ahc_handle_msg_reject(ahc, devinfo);
3626                /* FALLTHROUGH */
3627        case MSG_NOOP:
3628                done = MSGLOOP_MSGCOMPLETE;
3629                break;
3630        case MSG_EXTENDED:
3631        {
3632                /* Wait for enough of the message to begin validation */
3633                if (ahc->msgin_index < 2)
3634                        break;
3635                switch (ahc->msgin_buf[2]) {
3636                case MSG_EXT_SDTR:
3637                {
3638                        const struct ahc_syncrate *syncrate;
3639                        u_int         period;
3640                        u_int         ppr_options;
3641                        u_int         offset;
3642                        u_int         saved_offset;
3643                        
3644                        if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3645                                reject = TRUE;
3646                                break;
3647                        }
3648
3649                        /*
3650                         * Wait until we have both args before validating
3651                         * and acting on this message.
3652                         *
3653                         * Add one to MSG_EXT_SDTR_LEN to account for
3654                         * the extended message preamble.
3655                         */
3656                        if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3657                                break;
3658
3659                        period = ahc->msgin_buf[3];
3660                        ppr_options = 0;
3661                        saved_offset = offset = ahc->msgin_buf[4];
3662                        syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3663                                                           &ppr_options,
3664                                                           devinfo->role);
3665                        ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3666                                            targ_scsirate & WIDEXFER,
3667                                            devinfo->role);
3668                        if (bootverbose) {
3669                                printf("(%s:%c:%d:%d): Received "
3670                                       "SDTR period %x, offset %x\n\t"
3671                                       "Filtered to period %x, offset %x\n",
3672                                       ahc_name(ahc), devinfo->channel,
3673                                       devinfo->target, devinfo->lun,
3674                                       ahc->msgin_buf[3], saved_offset,
3675                                       period, offset);
3676                        }
3677                        ahc_set_syncrate(ahc, devinfo, 
3678                                         syncrate, period,
3679                                         offset, ppr_options,
3680                                         AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3681                                         /*paused*/TRUE);
3682
3683                        /*
3684                         * See if we initiated Sync Negotiation
3685                         * and didn't have to fall down to async
3686                         * transfers.
3687                         */
3688                        if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3689                                /* We started it */
3690                                if (saved_offset != offset) {
3691                                        /* Went too low - force async */
3692                                        reject = TRUE;
3693                                }
3694                        } else {
3695                                /*
3696                                 * Send our own SDTR in reply
3697                                 */
3698                                if (bootverbose
3699                                 && devinfo->role == ROLE_INITIATOR) {
3700                                        printf("(%s:%c:%d:%d): Target "
3701                                               "Initiated SDTR\n",
3702                                               ahc_name(ahc), devinfo->channel,
3703                                               devinfo->target, devinfo->lun);
3704                                }
3705                                ahc->msgout_index = 0;
3706                                ahc->msgout_len = 0;
3707                                ahc_construct_sdtr(ahc, devinfo,
3708                                                   period, offset);
3709                                ahc->msgout_index = 0;
3710                                response = TRUE;
3711                        }
3712                        done = MSGLOOP_MSGCOMPLETE;
3713                        break;
3714                }
3715                case MSG_EXT_WDTR:
3716                {
3717                        u_int bus_width;
3718                        u_int saved_width;
3719                        u_int sending_reply;
3720
3721                        sending_reply = FALSE;
3722                        if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3723                                reject = TRUE;
3724                                break;
3725                        }
3726
3727                        /*
3728                         * Wait until we have our arg before validating
3729                         * and acting on this message.
3730                         *
3731                         * Add one to MSG_EXT_WDTR_LEN to account for
3732                         * the extended message preamble.
3733                         */
3734                        if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3735                                break;
3736
3737                        bus_width = ahc->msgin_buf[3];
3738                        saved_width = bus_width;
3739                        ahc_validate_width(ahc, tinfo, &bus_width,
3740                                           devinfo->role);
3741                        if (bootverbose) {
3742                                printf("(%s:%c:%d:%d): Received WDTR "
3743                                       "%x filtered to %x\n",
3744                                       ahc_name(ahc), devinfo->channel,
3745                                       devinfo->target, devinfo->lun,
3746                                       saved_width, bus_width);
3747                        }
3748
3749                        if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3750                                /*
3751                                 * Don't send a WDTR back to the
3752                                 * target, since we asked first.
3753                                 * If the width went higher than our
3754                                 * request, reject it.
3755                                 */
3756                                if (saved_width > bus_width) {
3757                                        reject = TRUE;
3758                                        printf("(%s:%c:%d:%d): requested %dBit "
3759                                               "transfers.  Rejecting...\n",
3760                                               ahc_name(ahc), devinfo->channel,
3761                                               devinfo->target, devinfo->lun,
3762                                               8 * (0x01 << bus_width));
3763                                        bus_width = 0;
3764                                }
3765                        } else {
3766                                /*
3767                                 * Send our own WDTR in reply
3768                                 */
3769                                if (bootverbose
3770                                 && devinfo->role == ROLE_INITIATOR) {
3771                                        printf("(%s:%c:%d:%d): Target "
3772                                               "Initiated WDTR\n",
3773                                               ahc_name(ahc), devinfo->channel,
3774                                               devinfo->target, devinfo->lun);
3775                                }
3776                                ahc->msgout_index = 0;
3777                                ahc->msgout_len = 0;
3778                                ahc_construct_wdtr(ahc, devinfo, bus_width);
3779                                ahc->msgout_index = 0;
3780                                response = TRUE;
3781                                sending_reply = TRUE;
3782                        }
3783                        /*
3784                         * After a wide message, we are async, but
3785                         * some devices don't seem to honor this portion
3786                         * of the spec.  Force a renegotiation of the
3787                         * sync component of our transfer agreement even
3788                         * if our goal is async.  By updating our width
3789                         * after forcing the negotiation, we avoid
3790                         * renegotiating for width.
3791                         */
3792                        ahc_update_neg_request(ahc, devinfo, tstate,
3793                                               tinfo, AHC_NEG_ALWAYS);
3794                        ahc_set_width(ahc, devinfo, bus_width,
3795                                      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3796                                      /*paused*/TRUE);
3797                        if (sending_reply == FALSE && reject == FALSE) {
3798
3799                                /*
3800                                 * We will always have an SDTR to send.
3801                                 */
3802                                ahc->msgout_index = 0;
3803                                ahc->msgout_len = 0;
3804                                ahc_build_transfer_msg(ahc, devinfo);
3805                                ahc->msgout_index = 0;
3806                                response = TRUE;
3807                        }
3808                        done = MSGLOOP_MSGCOMPLETE;
3809                        break;
3810                }
3811                case MSG_EXT_PPR:
3812                {
3813                        const struct ahc_syncrate *syncrate;
3814                        u_int        period;
3815                        u_int        offset;
3816                        u_int        bus_width;
3817                        u_int        ppr_options;
3818                        u_int        saved_width;
3819                        u_int        saved_offset;
3820                        u_int        saved_ppr_options;
3821
3822                        if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3823                                reject = TRUE;
3824                                break;
3825                        }
3826
3827                        /*
3828                         * Wait until we have all args before validating
3829                         * and acting on this message.
3830                         *
3831                         * Add one to MSG_EXT_PPR_LEN to account for
3832                         * the extended message preamble.
3833                         */
3834                        if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3835                                break;
3836
3837                        period = ahc->msgin_buf[3];
3838                        offset = ahc->msgin_buf[5];
3839                        bus_width = ahc->msgin_buf[6];
3840                        saved_width = bus_width;
3841                        ppr_options = ahc->msgin_buf[7];
3842                        /*
3843                         * According to the spec, a DT only
3844                         * period factor with no DT option
3845                         * set implies async.
3846                         */
3847                        if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3848                         && period == 9)
3849                                offset = 0;
3850                        saved_ppr_options = ppr_options;
3851                        saved_offset = offset;
3852
3853                        /*
3854                         * Mask out any options we don't support
3855                         * on any controller.  Transfer options are
3856                         * only available if we are negotiating wide.
3857                         */
3858                        ppr_options &= MSG_EXT_PPR_DT_REQ;
3859                        if (bus_width == 0)
3860                                ppr_options = 0;
3861
3862                        ahc_validate_width(ahc, tinfo, &bus_width,
3863                                           devinfo->role);
3864                        syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3865                                                           &ppr_options,
3866                                                           devinfo->role);
3867                        ahc_validate_offset(ahc, tinfo, syncrate,
3868                                            &offset, bus_width,
3869                                            devinfo->role);
3870
3871                        if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3872                                /*
3873                                 * If we are unable to do any of the
3874                                 * requested options (we went too low),
3875                                 * then we'll have to reject the message.
3876                                 */
3877                                if (saved_width > bus_width
3878                                 || saved_offset != offset
3879                                 || saved_ppr_options != ppr_options) {
3880                                        reject = TRUE;
3881                                        period = 0;
3882                                        offset = 0;
3883                                        bus_width = 0;
3884                                        ppr_options = 0;
3885                                        syncrate = NULL;
3886                                }
3887                        } else {
3888                                if (devinfo->role != ROLE_TARGET)
3889                                        printf("(%s:%c:%d:%d): Target "
3890                                               "Initiated PPR\n",
3891                                               ahc_name(ahc), devinfo->channel,
3892                                               devinfo->target, devinfo->lun);
3893                                else
3894                                        printf("(%s:%c:%d:%d): Initiator "
3895                                               "Initiated PPR\n",
3896                                               ahc_name(ahc), devinfo->channel,
3897                                               devinfo->target, devinfo->lun);
3898                                ahc->msgout_index = 0;
3899                                ahc->msgout_len = 0;
3900                                ahc_construct_ppr(ahc, devinfo, period, offset,
3901                                                  bus_width, ppr_options);
3902                                ahc->msgout_index = 0;
3903                                response = TRUE;
3904                        }
3905                        if (bootverbose) {
3906                                printf("(%s:%c:%d:%d): Received PPR width %x, "
3907                                       "period %x, offset %x,options %x\n"
3908                                       "\tFiltered to width %x, period %x, "
3909                                       "offset %x, options %x\n",
3910                                       ahc_name(ahc), devinfo->channel,
3911                                       devinfo->target, devinfo->lun,
3912                                       saved_width, ahc->msgin_buf[3],
3913                                       saved_offset, saved_ppr_options,
3914                                       bus_width, period, offset, ppr_options);
3915                        }
3916                        ahc_set_width(ahc, devinfo, bus_width,
3917                                      AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3918                                      /*paused*/TRUE);
3919                        ahc_set_syncrate(ahc, devinfo,
3920                                         syncrate, period,
3921                                         offset, ppr_options,
3922                                         AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3923                                         /*paused*/TRUE);
3924                        done = MSGLOOP_MSGCOMPLETE;
3925                        break;
3926                }
3927                default:
3928                        /* Unknown extended message.  Reject it. */
3929                        reject = TRUE;
3930                        break;
3931                }
3932                break;
3933        }
3934#ifdef AHC_TARGET_MODE
3935        case MSG_BUS_DEV_RESET:
3936                ahc_handle_devreset(ahc, devinfo,
3937                                    CAM_BDR_SENT,
3938                                    "Bus Device Reset Received",
3939                                    /*verbose_level*/0);
3940                ahc_restart(ahc);
3941                done = MSGLOOP_TERMINATED;
3942                break;
3943        case MSG_ABORT_TAG:
3944        case MSG_ABORT:
3945        case MSG_CLEAR_QUEUE:
3946        {
3947                int tag;
3948
3949                /* Target mode messages */
3950                if (devinfo->role != ROLE_TARGET) {
3951                        reject = TRUE;
3952                        break;
3953                }
3954                tag = SCB_LIST_NULL;
3955                if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3956                        tag = ahc_inb(ahc, INITIATOR_TAG);
3957                ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3958                               devinfo->lun, tag, ROLE_TARGET,
3959                               CAM_REQ_ABORTED);
3960
3961                tstate = ahc->enabled_targets[devinfo->our_scsiid];
3962                if (tstate != NULL) {
3963                        struct ahc_tmode_lstate* lstate;
3964
3965                        lstate = tstate->enabled_luns[devinfo->lun];
3966                        if (lstate != NULL) {
3967                                ahc_queue_lstate_event(ahc, lstate,
3968                                                       devinfo->our_scsiid,
3969                                                       ahc->msgin_buf[0],
3970                                                       /*arg*/tag);
3971                                ahc_send_lstate_events(ahc, lstate);
3972                        }
3973                }
3974                ahc_restart(ahc);
3975                done = MSGLOOP_TERMINATED;
3976                break;
3977        }
3978#endif
3979        case MSG_TERM_IO_PROC:
3980        default:
3981                reject = TRUE;
3982                break;
3983        }
3984
3985        if (reject) {
3986                /*
3987                 * Setup to reject the message.
3988                 */
3989                ahc->msgout_index = 0;
3990                ahc->msgout_len = 1;
3991                ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3992                done = MSGLOOP_MSGCOMPLETE;
3993                response = TRUE;
3994        }
3995
3996        if (done != MSGLOOP_IN_PROG && !response)
3997                /* Clear the outgoing message buffer */
3998                ahc->msgout_len = 0;
3999
4000        return (done);
4001}
4002
4003/*
4004 * Process a message reject message.
4005 */
4006static int
4007ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
4008{
4009        /*
4010         * What we care about here is if we had an
4011         * outstanding SDTR or WDTR message for this
4012         * target.  If we did, this is a signal that
4013         * the target is refusing negotiation.
4014         */
4015        struct scb *scb;
4016        struct ahc_initiator_tinfo *tinfo;
4017        struct ahc_tmode_tstate *tstate;
4018        u_int scb_index;
4019        u_int last_msg;
4020        int   response = 0;
4021
4022        scb_index = ahc_inb(ahc, SCB_TAG);
4023        scb = ahc_lookup_scb(ahc, scb_index);
4024        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
4025                                    devinfo->our_scsiid,
4026                                    devinfo->target, &tstate);
4027        /* Might be necessary */
4028        last_msg = ahc_inb(ahc, LAST_MSG);
4029
4030        if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
4031                /*
4032                 * Target does not support the PPR message.
4033                 * Attempt to negotiate SPI-2 style.
4034                 */
4035                if (bootverbose) {
4036                        printf("(%s:%c:%d:%d): PPR Rejected. "
4037                               "Trying WDTR/SDTR\n",
4038                               ahc_name(ahc), devinfo->channel,
4039                               devinfo->target, devinfo->lun);
4040                }
4041                tinfo->goal.ppr_options = 0;
4042                tinfo->curr.transport_version = 2;
4043                tinfo->goal.transport_version = 2;
4044                ahc->msgout_index = 0;
4045                ahc->msgout_len = 0;
4046                ahc_build_transfer_msg(ahc, devinfo);
4047                ahc->msgout_index = 0;
4048                response = 1;
4049        } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
4050
4051                /* note 8bit xfers */
4052                printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
4053                       "8bit transfers\n", ahc_name(ahc),
4054                       devinfo->channel, devinfo->target, devinfo->lun);
4055                ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4056                              AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4057                              /*paused*/TRUE);
4058                /*
4059                 * No need to clear the sync rate.  If the target
4060                 * did not accept the command, our syncrate is
4061                 * unaffected.  If the target started the negotiation,
4062                 * but rejected our response, we already cleared the
4063                 * sync rate before sending our WDTR.
4064                 */
4065                if (tinfo->goal.offset != tinfo->curr.offset) {
4066
4067                        /* Start the sync negotiation */
4068                        ahc->msgout_index = 0;
4069                        ahc->msgout_len = 0;
4070                        ahc_build_transfer_msg(ahc, devinfo);
4071                        ahc->msgout_index = 0;
4072                        response = 1;
4073                }
4074        } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4075                /* note asynch xfers and clear flag */
4076                ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
4077                                 /*offset*/0, /*ppr_options*/0,
4078                                 AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4079                                 /*paused*/TRUE);
4080                printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
4081                       "Using asynchronous transfers\n",
4082                       ahc_name(ahc), devinfo->channel,
4083                       devinfo->target, devinfo->lun);
4084        } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4085                int tag_type;
4086                int mask;
4087
4088                tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4089
4090                if (tag_type == MSG_SIMPLE_TASK) {
4091                        printf("(%s:%c:%d:%d): refuses tagged commands.  "
4092                               "Performing non-tagged I/O\n", ahc_name(ahc),
4093                               devinfo->channel, devinfo->target, devinfo->lun);
4094                        ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_NONE);
4095                        mask = ~0x23;
4096                } else {
4097                        printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
4098                               "Performing simple queue tagged I/O only\n",
4099                               ahc_name(ahc), devinfo->channel, devinfo->target,
4100                               devinfo->lun, tag_type == MSG_ORDERED_TASK
4101                               ? "ordered" : "head of queue");
4102                        ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_BASIC);
4103                        mask = ~0x03;
4104                }
4105
4106                /*
4107                 * Resend the identify for this CCB as the target
4108                 * may believe that the selection is invalid otherwise.
4109                 */
4110                ahc_outb(ahc, SCB_CONTROL,
4111                         ahc_inb(ahc, SCB_CONTROL) & mask);
4112                 scb->hscb->control &= mask;
4113                ahc_set_transaction_tag(scb, /*enabled*/FALSE,
4114                                        /*type*/MSG_SIMPLE_TASK);
4115                ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
4116                ahc_assert_atn(ahc);
4117
4118                /*
4119                 * This transaction is now at the head of
4120                 * the untagged queue for this target.
4121                 */
4122                if ((ahc->flags & AHC_SCB_BTT) == 0) {
4123                        struct scb_tailq *untagged_q;
4124
4125                        untagged_q =
4126                            &(ahc->untagged_queues[devinfo->target_offset]);
4127                        TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
4128                        scb->flags |= SCB_UNTAGGEDQ;
4129                }
4130                ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4131                             scb->hscb->tag);
4132
4133                /*
4134                 * Requeue all tagged commands for this target
4135                 * currently in our posession so they can be
4136                 * converted to untagged commands.
4137                 */
4138                ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
4139                                   SCB_GET_CHANNEL(ahc, scb),
4140                                   SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4141                                   ROLE_INITIATOR, CAM_REQUEUE_REQ,
4142                                   SEARCH_COMPLETE);
4143        } else {
4144                /*
4145                 * Otherwise, we ignore it.
4146                 */
4147                printf("%s:%c:%d: Message reject for %x -- ignored\n",
4148                       ahc_name(ahc), devinfo->channel, devinfo->target,
4149                       last_msg);
4150        }
4151        return (response);
4152}
4153
4154/*
4155 * Process an ingnore wide residue message.
4156 */
4157static void
4158ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
4159{
4160        u_int scb_index;
4161        struct scb *scb;
4162
4163        scb_index = ahc_inb(ahc, SCB_TAG);
4164        scb = ahc_lookup_scb(ahc, scb_index);
4165        /*
4166         * XXX Actually check data direction in the sequencer?
4167         * Perhaps add datadir to some spare bits in the hscb?
4168         */
4169        if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
4170         || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
4171                /*
4172                 * Ignore the message if we haven't
4173                 * seen an appropriate data phase yet.
4174                 */
4175        } else {
4176                /*
4177                 * If the residual occurred on the last
4178                 * transfer and the transfer request was
4179                 * expected to end on an odd count, do
4180                 * nothing.  Otherwise, subtract a byte
4181                 * and update the residual count accordingly.
4182                 */
4183                uint32_t sgptr;
4184
4185                sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4186                if ((sgptr & SG_LIST_NULL) != 0
4187                 && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
4188                        /*
4189                         * If the residual occurred on the last
4190                         * transfer and the transfer request was
4191                         * expected to end on an odd count, do
4192                         * nothing.
4193                         */
4194                } else {
4195                        struct ahc_dma_seg *sg;
4196                        uint32_t data_cnt;
4197                        uint32_t data_addr;
4198                        uint32_t sglen;
4199
4200                        /* Pull in all of the sgptr */
4201                        sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
4202                        data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
4203
4204                        if ((sgptr & SG_LIST_NULL) != 0) {
4205                                /*
4206                                 * The residual data count is not updated
4207                                 * for the command run to completion case.
4208                                 * Explicitly zero the count.
4209                                 */
4210                                data_cnt &= ~AHC_SG_LEN_MASK;
4211                        }
4212
4213                        data_addr = ahc_inl(ahc, SHADDR);
4214
4215                        data_cnt += 1;
4216                        data_addr -= 1;
4217                        sgptr &= SG_PTR_MASK;
4218
4219                        sg = ahc_sg_bus_to_virt(scb, sgptr);
4220
4221                        /*
4222                         * The residual sg ptr points to the next S/G
4223                         * to load so we must go back one.
4224                         */
4225                        sg--;
4226                        sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
4227                        if (sg != scb->sg_list
4228                         && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
4229
4230                                sg--;
4231                                sglen = ahc_le32toh(sg->len);
4232                                /*
4233                                 * Preserve High Address and SG_LIST bits
4234                                 * while setting the count to 1.
4235                                 */
4236                                data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
4237                                data_addr = ahc_le32toh(sg->addr)
4238                                          + (sglen & AHC_SG_LEN_MASK) - 1;
4239
4240                                /*
4241                                 * Increment sg so it points to the
4242                                 * "next" sg.
4243                                 */
4244                                sg++;
4245                                sgptr = ahc_sg_virt_to_bus(scb, sg);
4246                        }
4247                        ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
4248                        ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
4249                        /*
4250                         * Toggle the "oddness" of the transfer length
4251                         * to handle this mid-transfer ignore wide
4252                         * residue.  This ensures that the oddness is
4253                         * correct for subsequent data transfers.
4254                         */
4255                        ahc_outb(ahc, SCB_LUN,
4256                                 ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
4257                }
4258        }
4259}
4260
4261
4262/*
4263 * Reinitialize the data pointers for the active transfer
4264 * based on its current residual.
4265 */
4266static void
4267ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
4268{
4269        struct         scb *scb;
4270        struct         ahc_dma_seg *sg;
4271        u_int         scb_index;
4272        uint32_t sgptr;
4273        uint32_t resid;
4274        uint32_t dataptr;
4275
4276        scb_index = ahc_inb(ahc, SCB_TAG);
4277        scb = ahc_lookup_scb(ahc, scb_index);
4278        sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
4279              | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
4280              | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
4281              |        ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4282
4283        sgptr &= SG_PTR_MASK;
4284        sg = ahc_sg_bus_to_virt(scb, sgptr);
4285
4286        /* The residual sg_ptr always points to the next sg */
4287        sg--;
4288
4289        resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
4290              | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
4291              | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
4292
4293        dataptr = ahc_le32toh(sg->addr)
4294                + (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
4295                - resid;
4296        if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
4297                u_int dscommand1;
4298
4299                dscommand1 = ahc_inb(ahc, DSCOMMAND1);
4300                ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
4301                ahc_outb(ahc, HADDR,
4302                         (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
4303                ahc_outb(ahc, DSCOMMAND1, dscommand1);
4304        }
4305        ahc_outb(ahc, HADDR + 3, dataptr >> 24);
4306        ahc_outb(ahc, HADDR + 2, dataptr >> 16);
4307        ahc_outb(ahc, HADDR + 1, dataptr >> 8);
4308        ahc_outb(ahc, HADDR, dataptr);
4309        ahc_outb(ahc, HCNT + 2, resid >> 16);
4310        ahc_outb(ahc, HCNT + 1, resid >> 8);
4311        ahc_outb(ahc, HCNT, resid);
4312        if ((ahc->features & AHC_ULTRA2) == 0) {
4313                ahc_outb(ahc, STCNT + 2, resid >> 16);
4314                ahc_outb(ahc, STCNT + 1, resid >> 8);
4315                ahc_outb(ahc, STCNT, resid);
4316        }
4317}
4318
4319/*
4320 * Handle the effects of issuing a bus device reset message.
4321 */
4322static void
4323ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4324                    cam_status status, char *message, int verbose_level)
4325{
4326#ifdef AHC_TARGET_MODE
4327        struct ahc_tmode_tstate* tstate;
4328        u_int lun;
4329#endif
4330        int found;
4331
4332        found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
4333                               CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
4334                               status);
4335
4336#ifdef AHC_TARGET_MODE
4337        /*
4338         * Send an immediate notify ccb to all target mord peripheral
4339         * drivers affected by this action.
4340         */
4341        tstate = ahc->enabled_targets[devinfo->our_scsiid];
4342        if (tstate != NULL) {
4343                for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
4344                        struct ahc_tmode_lstate* lstate;
4345
4346                        lstate = tstate->enabled_luns[lun];
4347                        if (lstate == NULL)
4348                                continue;
4349
4350                        ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
4351                                               MSG_BUS_DEV_RESET, /*arg*/0);
4352                        ahc_send_lstate_events(ahc, lstate);
4353                }
4354        }
4355#endif
4356
4357        /*
4358         * Go back to async/narrow transfers and renegotiate.
4359         */
4360        ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4361                      AHC_TRANS_CUR, /*paused*/TRUE);
4362        ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
4363                         /*period*/0, /*offset*/0, /*ppr_options*/0,
4364                         AHC_TRANS_CUR, /*paused*/TRUE);
4365        
4366        if (status != CAM_SEL_TIMEOUT)
4367                ahc_send_async(ahc, devinfo->channel, devinfo->target,
4368                               CAM_LUN_WILDCARD, AC_SENT_BDR);
4369
4370        if (message != NULL
4371         && (verbose_level <= bootverbose))
4372                printf("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
4373                       message, devinfo->channel, devinfo->target, found);
4374}
4375
4376#ifdef AHC_TARGET_MODE
4377static void
4378ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4379                       struct scb *scb)
4380{
4381
4382        /*              
4383         * To facilitate adding multiple messages together,
4384         * each routine should increment the index and len
4385         * variables instead of setting them explicitly.
4386         */             
4387        ahc->msgout_index = 0;
4388        ahc->msgout_len = 0;
4389
4390        if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
4391                ahc_build_transfer_msg(ahc, devinfo);
4392        else
4393                panic("ahc_intr: AWAITING target message with no message");
4394
4395        ahc->msgout_index = 0;
4396        ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
4397}
4398#endif
4399/**************************** Initialization **********************************/
4400/*
4401 * Allocate a controller structure for a new device
4402 * and perform initial initializion.
4403 */
4404struct ahc_softc *
4405ahc_alloc(void *platform_arg, char *name)
4406{
4407        struct  ahc_softc *ahc;
4408        int        i;
4409
4410#ifndef        __FreeBSD__
4411        ahc = malloc(sizeof(*ahc), M_DEVBUF, M_NOWAIT);
4412        if (!ahc) {
4413                printf("aic7xxx: cannot malloc softc!\n");
4414                free(name, M_DEVBUF);
4415                return NULL;
4416        }
4417#else
4418        ahc = device_get_softc((device_t)platform_arg);
4419#endif
4420        memset(ahc, 0, sizeof(*ahc));
4421        ahc->seep_config = malloc(sizeof(*ahc->seep_config),
4422                                  M_DEVBUF, M_NOWAIT);
4423        if (ahc->seep_config == NULL) {
4424#ifndef        __FreeBSD__
4425                free(ahc, M_DEVBUF);
4426#endif
4427                free(name, M_DEVBUF);
4428                return (NULL);
4429        }
4430        LIST_INIT(&ahc->pending_scbs);
4431        /* We don't know our unit number until the OSM sets it */
4432        ahc->name = name;
4433        ahc->unit = -1;
4434        ahc->description = NULL;
4435        ahc->channel = 'A';
4436        ahc->channel_b = 'B';
4437        ahc->chip = AHC_NONE;
4438        ahc->features = AHC_FENONE;
4439        ahc->bugs = AHC_BUGNONE;
4440        ahc->flags = AHC_FNONE;
4441        /*
4442         * Default to all error reporting enabled with the
4443         * sequencer operating at its fastest speed.
4444         * The bus attach code may modify this.
4445         */
4446        ahc->seqctl = FASTMODE;
4447
4448        for (i = 0; i < AHC_NUM_TARGETS; i++)
4449                TAILQ_INIT(&ahc->untagged_queues[i]);
4450        if (ahc_platform_alloc(ahc, platform_arg) != 0) {
4451                ahc_free(ahc);
4452                ahc = NULL;
4453        }
4454        return (ahc);
4455}
4456
4457int
4458ahc_softc_init(struct ahc_softc *ahc)
4459{
4460
4461        /* The IRQMS bit is only valid on VL and EISA chips */
4462        if ((ahc->chip & AHC_PCI) == 0)
4463                ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
4464        else
4465                ahc->unpause = 0;
4466        ahc->pause = ahc->unpause | PAUSE; 
4467        /* XXX The shared scb data stuff should be deprecated */
4468        if (ahc->scb_data == NULL) {
4469                ahc->scb_data = malloc(sizeof(*ahc->scb_data),
4470                                       M_DEVBUF, M_NOWAIT);
4471                if (ahc->scb_data == NULL)
4472                        return (ENOMEM);
4473                memset(ahc->scb_data, 0, sizeof(*ahc->scb_data));
4474        }
4475
4476        return (0);
4477}
4478
4479void
4480ahc_set_unit(struct ahc_softc *ahc, int unit)
4481{
4482        ahc->unit = unit;
4483}
4484
4485void
4486ahc_set_name(struct ahc_softc *ahc, char *name)
4487{
4488        if (ahc->name != NULL)
4489                free(ahc->name, M_DEVBUF);
4490        ahc->name = name;
4491}
4492
4493void
4494ahc_free(struct ahc_softc *ahc)
4495{
4496        int i;
4497
4498        switch (ahc->init_level) {
4499        default:
4500        case 5:
4501                ahc_shutdown(ahc);
4502                /* FALLTHROUGH */
4503        case 4:
4504                ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
4505                                  ahc->shared_data_dmamap);
4506                /* FALLTHROUGH */
4507        case 3:
4508                ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
4509                                ahc->shared_data_dmamap);
4510                ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
4511                                   ahc->shared_data_dmamap);
4512                /* FALLTHROUGH */
4513        case 2:
4514                ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
4515        case 1:
4516#ifndef __linux__
4517                ahc_dma_tag_destroy(ahc, ahc->buffer_dmat);
4518#endif
4519                break;
4520        case 0:
4521                break;
4522        }
4523
4524#ifndef __linux__
4525        ahc_dma_tag_destroy(ahc, ahc->parent_dmat);
4526#endif
4527        ahc_platform_free(ahc);
4528        ahc_fini_scbdata(ahc);
4529        for (i = 0; i < AHC_NUM_TARGETS; i++) {
4530                struct ahc_tmode_tstate *tstate;
4531
4532                tstate = ahc->enabled_targets[i];
4533                if (tstate != NULL) {
4534#ifdef AHC_TARGET_MODE
4535                        int j;
4536
4537                        for (j = 0; j < AHC_NUM_LUNS; j++) {
4538                                struct ahc_tmode_lstate *lstate;
4539
4540                                lstate = tstate->enabled_luns[j];
4541                                if (lstate != NULL) {
4542                                        xpt_free_path(lstate->path);
4543                                        free(lstate, M_DEVBUF);
4544                                }
4545                        }
4546#endif
4547                        free(tstate, M_DEVBUF);
4548                }
4549        }
4550#ifdef AHC_TARGET_MODE
4551        if (ahc->black_hole != NULL) {
4552                xpt_free_path(ahc->black_hole->path);
4553                free(ahc->black_hole, M_DEVBUF);
4554        }
4555#endif
4556        if (ahc->name != NULL)
4557                free(ahc->name, M_DEVBUF);
4558        if (ahc->seep_config != NULL)
4559                free(ahc->seep_config, M_DEVBUF);
4560#ifndef __FreeBSD__
4561        free(ahc, M_DEVBUF);
4562#endif
4563        return;
4564}
4565
4566static void
4567ahc_shutdown(void *arg)
4568{
4569        struct        ahc_softc *ahc;
4570        int        i;
4571
4572        ahc = (struct ahc_softc *)arg;
4573
4574        /* This will reset most registers to 0, but not all */
4575        ahc_reset(ahc, /*reinit*/FALSE);
4576        ahc_outb(ahc, SCSISEQ, 0);
4577        ahc_outb(ahc, SXFRCTL0, 0);
4578        ahc_outb(ahc, DSPCISTATUS, 0);
4579
4580        for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4581                ahc_outb(ahc, i, 0);
4582}
4583
4584/*
4585 * Reset the controller and record some information about it
4586 * that is only available just after a reset.  If "reinit" is
4587 * non-zero, this reset occured after initial configuration
4588 * and the caller requests that the chip be fully reinitialized
4589 * to a runable state.  Chip interrupts are *not* enabled after
4590 * a reinitialization.  The caller must enable interrupts via
4591 * ahc_intr_enable().
4592 */
4593int
4594ahc_reset(struct ahc_softc *ahc, int reinit)
4595{
4596        u_int        sblkctl;
4597        u_int        sxfrctl1_a, sxfrctl1_b;
4598        int        error;
4599        int        wait;
4600        
4601        /*
4602         * Preserve the value of the SXFRCTL1 register for all channels.
4603         * It contains settings that affect termination and we don't want
4604         * to disturb the integrity of the bus.
4605         */
4606        ahc_pause(ahc);
4607        sxfrctl1_b = 0;
4608        if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4609                u_int sblkctl;
4610
4611                /*
4612                 * Save channel B's settings in case this chip
4613                 * is setup for TWIN channel operation.
4614                 */
4615                sblkctl = ahc_inb(ahc, SBLKCTL);
4616                ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4617                sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4618                ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4619        }
4620        sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4621
4622        ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4623
4624        /*
4625         * Ensure that the reset has finished.  We delay 1000us
4626         * prior to reading the register to make sure the chip
4627         * has sufficiently completed its reset to handle register
4628         * accesses.
4629         */
4630        wait = 1000;
4631        do {
4632                ahc_delay(1000);
4633        } while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4634
4635        if (wait == 0) {
4636                printf("%s: WARNING - Failed chip reset!  "
4637                       "Trying to initialize anyway.\n", ahc_name(ahc));
4638        }
4639        ahc_outb(ahc, HCNTRL, ahc->pause);
4640
4641        /* Determine channel configuration */
4642        sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4643        /* No Twin Channel PCI cards */
4644        if ((ahc->chip & AHC_PCI) != 0)
4645                sblkctl &= ~SELBUSB;
4646        switch (sblkctl) {
4647        case 0:
4648                /* Single Narrow Channel */
4649                break;
4650        case 2:
4651                /* Wide Channel */
4652                ahc->features |= AHC_WIDE;
4653                break;
4654        case 8:
4655                /* Twin Channel */
4656                ahc->features |= AHC_TWIN;
4657                break;
4658        default:
4659                printf(" Unsupported adapter type.  Ignoring\n");
4660                return(-1);
4661        }
4662
4663        /*
4664         * Reload sxfrctl1.
4665         *
4666         * We must always initialize STPWEN to 1 before we
4667         * restore the saved values.  STPWEN is initialized
4668         * to a tri-state condition which can only be cleared
4669         * by turning it on.
4670         */
4671        if ((ahc->features & AHC_TWIN) != 0) {
4672                u_int sblkctl;
4673
4674                sblkctl = ahc_inb(ahc, SBLKCTL);
4675                ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4676                ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4677                ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4678        }
4679        ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4680
4681        error = 0;
4682        if (reinit != 0)
4683                /*
4684                 * If a recovery action has forced a chip reset,
4685                 * re-initialize the chip to our liking.
4686                 */
4687                error = ahc->bus_chip_init(ahc);
4688#ifdef AHC_DUMP_SEQ
4689        else 
4690                ahc_dumpseq(ahc);
4691#endif
4692
4693        return (error);
4694}
4695
4696/*
4697 * Determine the number of SCBs available on the controller
4698 */
4699int
4700ahc_probe_scbs(struct ahc_softc *ahc) {
4701        int i;
4702
4703        for (i = 0; i < AHC_SCB_MAX; i++) {
4704
4705                ahc_outb(ahc, SCBPTR, i);
4706                ahc_outb(ahc, SCB_BASE, i);
4707                if (ahc_inb(ahc, SCB_BASE) != i)
4708                        break;
4709                ahc_outb(ahc, SCBPTR, 0);
4710                if (ahc_inb(ahc, SCB_BASE) != 0)
4711                        break;
4712        }
4713        return (i);
4714}
4715
4716static void
4717ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 
4718{
4719        dma_addr_t *baddr;
4720
4721        baddr = (dma_addr_t *)arg;
4722        *baddr = segs->ds_addr;
4723}
4724
4725static void
4726ahc_build_free_scb_list(struct ahc_softc *ahc)
4727{
4728        int scbsize;
4729        int i;
4730
4731        scbsize = 32;
4732        if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4733                scbsize = 64;
4734
4735        for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4736                int j;
4737
4738                ahc_outb(ahc, SCBPTR, i);
4739
4740                /*
4741                 * Touch all SCB bytes to avoid parity errors
4742                 * should one of our debugging routines read
4743                 * an otherwise uninitiatlized byte.
4744                 */
4745                for (j = 0; j < scbsize; j++)
4746                        ahc_outb(ahc, SCB_BASE+j, 0xFF);
4747
4748                /* Clear the control byte. */
4749                ahc_outb(ahc, SCB_CONTROL, 0);
4750
4751                /* Set the next pointer */
4752                if ((ahc->flags & AHC_PAGESCBS) != 0)
4753                        ahc_outb(ahc, SCB_NEXT, i+1);
4754                else 
4755                        ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4756
4757                /* Make the tag number, SCSIID, and lun invalid */
4758                ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4759                ahc_outb(ahc, SCB_SCSIID, 0xFF);
4760                ahc_outb(ahc, SCB_LUN, 0xFF);
4761        }
4762
4763        if ((ahc->flags & AHC_PAGESCBS) != 0) {
4764                /* SCB 0 heads the free list. */
4765                ahc_outb(ahc, FREE_SCBH, 0);
4766        } else {
4767                /* No free list. */
4768                ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4769        }
4770
4771        /* Make sure that the last SCB terminates the free list */
4772        ahc_outb(ahc, SCBPTR, i-1);
4773        ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4774}
4775
4776static int
4777ahc_init_scbdata(struct ahc_softc *ahc)
4778{
4779        struct scb_data *scb_data;
4780
4781        scb_data = ahc->scb_data;
4782        SLIST_INIT(&scb_data->free_scbs);
4783        SLIST_INIT(&scb_data->sg_maps);
4784
4785        /* Allocate SCB resources */
4786        scb_data->scbarray =
4787            (struct scb *)malloc(sizeof(struct scb) * AHC_SCB_MAX_ALLOC,
4788                                 M_DEVBUF, M_NOWAIT);
4789        if (scb_data->scbarray == NULL)
4790                return (ENOMEM);
4791        memset(scb_data->scbarray, 0, sizeof(struct scb) * AHC_SCB_MAX_ALLOC);
4792
4793        /* Determine the number of hardware SCBs and initialize them */
4794
4795        scb_data->maxhscbs = ahc_probe_scbs(ahc);
4796        if (ahc->scb_data->maxhscbs == 0) {
4797                printf("%s: No SCB space found\n", ahc_name(ahc));
4798                return (ENXIO);
4799        }
4800
4801        /*
4802         * Create our DMA tags.  These tags define the kinds of device
4803         * accessible memory allocations and memory mappings we will
4804         * need to perform during normal operation.
4805         *
4806         * Unless we need to further restrict the allocation, we rely
4807         * on the restrictions of the parent dmat, hence the common
4808         * use of MAXADDR and MAXSIZE.
4809         */
4810
4811        /* DMA tag for our hardware scb structures */
4812        if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4813                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4814                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4815                               /*highaddr*/BUS_SPACE_MAXADDR,
4816                               /*filter*/NULL, /*filterarg*/NULL,
4817                               AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4818                               /*nsegments*/1,
4819                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4820                               /*flags*/0, &scb_data->hscb_dmat) != 0) {
4821                goto error_exit;
4822        }
4823
4824        scb_data->init_level++;
4825
4826        /* Allocation for our hscbs */
4827        if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4828                             (void **)&scb_data->hscbs,
4829                             BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4830                goto error_exit;
4831        }
4832
4833        scb_data->init_level++;
4834
4835        /* And permanently map them */
4836        ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4837                        scb_data->hscbs,
4838                        AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4839                        ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4840
4841        scb_data->init_level++;
4842
4843        /* DMA tag for our sense buffers */
4844        if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4845                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4846                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4847                               /*highaddr*/BUS_SPACE_MAXADDR,
4848                               /*filter*/NULL, /*filterarg*/NULL,
4849                               AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4850                               /*nsegments*/1,
4851                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4852                               /*flags*/0, &scb_data->sense_dmat) != 0) {
4853                goto error_exit;
4854        }
4855
4856        scb_data->init_level++;
4857
4858        /* Allocate them */
4859        if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4860                             (void **)&scb_data->sense,
4861                             BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4862                goto error_exit;
4863        }
4864
4865        scb_data->init_level++;
4866
4867        /* And permanently map them */
4868        ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4869                        scb_data->sense,
4870                        AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4871                        ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4872
4873        scb_data->init_level++;
4874
4875        /* DMA tag for our S/G structures.  We allocate in page sized chunks */
4876        if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4877                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4878                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4879                               /*highaddr*/BUS_SPACE_MAXADDR,
4880                               /*filter*/NULL, /*filterarg*/NULL,
4881                               PAGE_SIZE, /*nsegments*/1,
4882                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4883                               /*flags*/0, &scb_data->sg_dmat) != 0) {
4884                goto error_exit;
4885        }
4886
4887        scb_data->init_level++;
4888
4889        /* Perform initial CCB allocation */
4890        memset(scb_data->hscbs, 0,
4891               AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4892        ahc_alloc_scbs(ahc);
4893
4894        if (scb_data->numscbs == 0) {
4895                printf("%s: ahc_init_scbdata - "
4896                       "Unable to allocate initial scbs\n",
4897                       ahc_name(ahc));
4898                goto error_exit;
4899        }
4900
4901        /*
4902         * Reserve the next queued SCB.
4903         */
4904        ahc->next_queued_scb = ahc_get_scb(ahc);
4905
4906        /*
4907         * Note that we were successfull
4908         */
4909        return (0); 
4910
4911error_exit:
4912
4913        return (ENOMEM);
4914}
4915
4916static void
4917ahc_fini_scbdata(struct ahc_softc *ahc)
4918{
4919        struct scb_data *scb_data;
4920
4921        scb_data = ahc->scb_data;
4922        if (scb_data == NULL)
4923                return;
4924
4925        switch (scb_data->init_level) {
4926        default:
4927        case 7:
4928        {
4929                struct sg_map_node *sg_map;
4930
4931                while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4932                        SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4933                        ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4934                                          sg_map->sg_dmamap);
4935                        ahc_dmamem_free(ahc, scb_data->sg_dmat,
4936                                        sg_map->sg_vaddr,
4937                                        sg_map->sg_dmamap);
4938                        free(sg_map, M_DEVBUF);
4939                }
4940                ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4941        }
4942        case 6:
4943                ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4944                                  scb_data->sense_dmamap);
4945        case 5:
4946                ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4947                                scb_data->sense_dmamap);
4948                ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4949                                   scb_data->sense_dmamap);
4950        case 4:
4951                ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4952        case 3:
4953                ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4954                                  scb_data->hscb_dmamap);
4955        case 2:
4956                ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4957                                scb_data->hscb_dmamap);
4958                ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4959                                   scb_data->hscb_dmamap);
4960        case 1:
4961                ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4962                break;
4963        case 0:
4964                break;
4965        }
4966        if (scb_data->scbarray != NULL)
4967                free(scb_data->scbarray, M_DEVBUF);
4968}
4969
4970static void
4971ahc_alloc_scbs(struct ahc_softc *ahc)
4972{
4973        struct scb_data *scb_data;
4974        struct scb *next_scb;
4975        struct sg_map_node *sg_map;
4976        dma_addr_t physaddr;
4977        struct ahc_dma_seg *segs;
4978        int newcount;
4979        int i;
4980
4981        scb_data = ahc->scb_data;
4982        if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4983                /* Can't allocate any more */
4984                return;
4985
4986        next_scb = &scb_data->scbarray[scb_data->numscbs];
4987
4988        sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
4989
4990        if (sg_map == NULL)
4991                return;
4992
4993        /* Allocate S/G space for the next batch of SCBS */
4994        if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4995                             (void **)&sg_map->sg_vaddr,
4996                             BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4997                free(sg_map, M_DEVBUF);
4998                return;
4999        }
5000
5001        SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
5002
5003        ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
5004                        sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
5005                        &sg_map->sg_physaddr, /*flags*/0);
5006
5007        segs = sg_map->sg_vaddr;
5008        physaddr = sg_map->sg_physaddr;
5009
5010        newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
5011        newcount = min(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
5012        for (i = 0; i < newcount; i++) {
5013                struct scb_platform_data *pdata;
5014#ifndef __linux__
5015                int error;
5016#endif
5017                pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
5018                                                           M_DEVBUF, M_NOWAIT);
5019                if (pdata == NULL)
5020                        break;
5021                next_scb->platform_data = pdata;
5022                next_scb->sg_map = sg_map;
5023                next_scb->sg_list = segs;
5024                /*
5025                 * The sequencer always starts with the second entry.
5026                 * The first entry is embedded in the scb.
5027                 */
5028                next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
5029                next_scb->ahc_softc = ahc;
5030                next_scb->flags = SCB_FREE;
5031#ifndef __linux__
5032                error = ahc_dmamap_create(ahc, ahc->buffer_dmat, /*flags*/0,
5033                                          &next_scb->dmamap);
5034                if (error != 0)
5035                        break;
5036#endif
5037                next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
5038                next_scb->hscb->tag = ahc->scb_data->numscbs;
5039                SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
5040                                  next_scb, links.sle);
5041                segs += AHC_NSEG;
5042                physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
5043                next_scb++;
5044                ahc->scb_data->numscbs++;
5045        }
5046}
5047
5048void
5049ahc_controller_info(struct ahc_softc *ahc, char *buf)
5050{
5051        int len;
5052
5053        len = sprintf(buf, "%s: ", ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
5054        buf += len;
5055        if ((ahc->features & AHC_TWIN) != 0)
5056                 len = sprintf(buf, "Twin Channel, A SCSI Id=%d, "
5057                              "B SCSI Id=%d, primary %c, ",
5058                              ahc->our_id, ahc->our_id_b,
5059                              (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
5060        else {
5061                const char *speed;
5062                const char *type;
5063
5064                speed = "";
5065                if ((ahc->features & AHC_ULTRA) != 0) {
5066                        speed = "Ultra ";
5067                } else if ((ahc->features & AHC_DT) != 0) {
5068                        speed = "Ultra160 ";
5069                } else if ((ahc->features & AHC_ULTRA2) != 0) {
5070                        speed = "Ultra2 ";
5071                }
5072                if ((ahc->features & AHC_WIDE) != 0) {
5073                        type = "Wide";
5074                } else {
5075                        type = "Single";
5076                }
5077                len = sprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
5078                              speed, type, ahc->channel, ahc->our_id);
5079        }
5080        buf += len;
5081
5082        if ((ahc->flags & AHC_PAGESCBS) != 0)
5083                sprintf(buf, "%d/%d SCBs",
5084                        ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
5085        else
5086                sprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
5087}
5088
5089int
5090ahc_chip_init(struct ahc_softc *ahc)
5091{
5092        int         term;
5093        int         error;
5094        u_int         i;
5095        u_int         scsi_conf;
5096        u_int         scsiseq_template;
5097        uint32_t physaddr;
5098
5099        ahc_outb(ahc, SEQ_FLAGS, 0);
5100        ahc_outb(ahc, SEQ_FLAGS2, 0);
5101
5102        /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
5103        if (ahc->features & AHC_TWIN) {
5104
5105                /*
5106                 * Setup Channel B first.
5107                 */
5108                ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
5109                term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
5110                ahc_outb(ahc, SCSIID, ahc->our_id_b);
5111                scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5112                ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5113                                        |term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
5114                if ((ahc->features & AHC_ULTRA2) != 0)
5115                        ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5116                ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5117                ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5118
5119                /* Select Channel A */
5120                ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
5121        }
5122        term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
5123        if ((ahc->features & AHC_ULTRA2) != 0)
5124                ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
5125        else
5126                ahc_outb(ahc, SCSIID, ahc->our_id);
5127        scsi_conf = ahc_inb(ahc, SCSICONF);
5128        ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5129                                |term|ahc->seltime
5130                                |ENSTIMER|ACTNEGEN);
5131        if ((ahc->features & AHC_ULTRA2) != 0)
5132                ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5133        ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5134        ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5135
5136        /* There are no untagged SCBs active yet. */
5137        for (i = 0; i < 16; i++) {
5138                ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
5139                if ((ahc->flags & AHC_SCB_BTT) != 0) {
5140                        int lun;
5141
5142                        /*
5143                         * The SCB based BTT allows an entry per
5144                         * target and lun pair.
5145                         */
5146                        for (lun = 1; lun < AHC_NUM_LUNS; lun++)
5147                                ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
5148                }
5149        }
5150
5151        /* All of our queues are empty */
5152        for (i = 0; i < 256; i++)
5153                ahc->qoutfifo[i] = SCB_LIST_NULL;
5154        ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
5155
5156        for (i = 0; i < 256; i++)
5157                ahc->qinfifo[i] = SCB_LIST_NULL;
5158
5159        if ((ahc->features & AHC_MULTI_TID) != 0) {
5160                ahc_outb(ahc, TARGID, 0);
5161                ahc_outb(ahc, TARGID + 1, 0);
5162        }
5163
5164        /*
5165         * Tell the sequencer where it can find our arrays in memory.
5166         */
5167        physaddr = ahc->scb_data->hscb_busaddr;
5168        ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
5169        ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
5170        ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
5171        ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
5172
5173        physaddr = ahc->shared_data_busaddr;
5174        ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
5175        ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
5176        ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
5177        ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
5178
5179        /*
5180         * Initialize the group code to command length table.
5181         * This overrides the values in TARG_SCSIRATE, so only
5182         * setup the table after we have processed that information.
5183         */
5184        ahc_outb(ahc, CMDSIZE_TABLE, 5);
5185        ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
5186        ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
5187        ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
5188        ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
5189        ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
5190        ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
5191        ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
5192                
5193        if ((ahc->features & AHC_HS_MAILBOX) != 0)
5194                ahc_outb(ahc, HS_MAILBOX, 0);
5195
5196        /* Tell the sequencer of our initial queue positions */
5197        if ((ahc->features & AHC_TARGETMODE) != 0) {
5198                ahc->tqinfifonext = 1;
5199                ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
5200                ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
5201        }
5202        ahc->qinfifonext = 0;
5203        ahc->qoutfifonext = 0;
5204        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5205                ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
5206                ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5207                ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
5208                ahc_outb(ahc, SDSCB_QOFF, 0);
5209        } else {
5210                ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5211                ahc_outb(ahc, QINPOS, ahc->qinfifonext);
5212                ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
5213        }
5214
5215        /* We don't have any waiting selections */
5216        ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
5217
5218        /* Our disconnection list is empty too */
5219        ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
5220
5221        /* Message out buffer starts empty */
5222        ahc_outb(ahc, MSG_OUT, MSG_NOOP);
5223
5224        /*
5225         * Setup the allowed SCSI Sequences based on operational mode.
5226         * If we are a target, we'll enalbe select in operations once
5227         * we've had a lun enabled.
5228         */
5229        scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
5230        if ((ahc->flags & AHC_INITIATORROLE) != 0)
5231                scsiseq_template |= ENRSELI;
5232        ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
5233
5234        /* Initialize our list of free SCBs. */
5235        ahc_build_free_scb_list(ahc);
5236
5237        /*
5238         * Tell the sequencer which SCB will be the next one it receives.
5239         */
5240        ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5241
5242        /*
5243         * Load the Sequencer program and Enable the adapter
5244         * in "fast" mode.
5245         */
5246        if (bootverbose)
5247                printf("%s: Downloading Sequencer Program...",
5248                       ahc_name(ahc));
5249
5250        error = ahc_loadseq(ahc);
5251        if (error != 0)
5252                return (error);
5253
5254        if ((ahc->features & AHC_ULTRA2) != 0) {
5255                int wait;
5256
5257                /*
5258                 * Wait for up to 500ms for our transceivers
5259                 * to settle.  If the adapter does not have
5260                 * a cable attached, the transceivers may
5261                 * never settle, so don't complain if we
5262                 * fail here.
5263                 */
5264                for (wait = 5000;
5265                     (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
5266                     wait--)
5267                        ahc_delay(100);
5268        }
5269        ahc_restart(ahc);
5270        return (0);
5271}
5272
5273/*
5274 * Start the board, ready for normal operation
5275 */
5276int
5277ahc_init(struct ahc_softc *ahc)
5278{
5279        int         max_targ;
5280        u_int         i;
5281        u_int         scsi_conf;
5282        u_int         ultraenb;
5283        u_int         discenable;
5284        u_int         tagenable;
5285        size_t         driver_data_size;
5286
5287#ifdef AHC_DEBUG
5288        if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
5289                ahc->flags |= AHC_SEQUENCER_DEBUG;
5290#endif
5291
5292#ifdef AHC_PRINT_SRAM
5293        printf("Scratch Ram:");
5294        for (i = 0x20; i < 0x5f; i++) {
5295                if (((i % 8) == 0) && (i != 0)) {
5296                        printf ("\n              ");
5297                }
5298                printf (" 0x%x", ahc_inb(ahc, i));
5299        }
5300        if ((ahc->features & AHC_MORE_SRAM) != 0) {
5301                for (i = 0x70; i < 0x7f; i++) {
5302                        if (((i % 8) == 0) && (i != 0)) {
5303                                printf ("\n              ");
5304                        }
5305                        printf (" 0x%x", ahc_inb(ahc, i));
5306                }
5307        }
5308        printf ("\n");
5309        /*
5310         * Reading uninitialized scratch ram may
5311         * generate parity errors.
5312         */
5313        ahc_outb(ahc, CLRINT, CLRPARERR);
5314        ahc_outb(ahc, CLRINT, CLRBRKADRINT);
5315#endif
5316        max_targ = 15;
5317
5318        /*
5319         * Assume we have a board at this stage and it has been reset.
5320         */
5321        if ((ahc->flags & AHC_USEDEFAULTS) != 0)
5322                ahc->our_id = ahc->our_id_b = 7;
5323        
5324        /*
5325         * Default to allowing initiator operations.
5326         */
5327        ahc->flags |= AHC_INITIATORROLE;
5328
5329        /*
5330         * Only allow target mode features if this unit has them enabled.
5331         */
5332        if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
5333                ahc->features &= ~AHC_TARGETMODE;
5334
5335#ifndef __linux__
5336        /* DMA tag for mapping buffers into device visible space. */
5337        if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5338                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5339                               /*lowaddr*/ahc->flags & AHC_39BIT_ADDRESSING
5340                                        ? (dma_addr_t)0x7FFFFFFFFFULL
5341                                        : BUS_SPACE_MAXADDR_32BIT,
5342                               /*highaddr*/BUS_SPACE_MAXADDR,
5343                               /*filter*/NULL, /*filterarg*/NULL,
5344                               /*maxsize*/(AHC_NSEG - 1) * PAGE_SIZE,
5345                               /*nsegments*/AHC_NSEG,
5346                               /*maxsegsz*/AHC_MAXTRANSFER_SIZE,
5347                               /*flags*/BUS_DMA_ALLOCNOW,
5348                               &ahc->buffer_dmat) != 0) {
5349                return (ENOMEM);
5350        }
5351#endif
5352
5353        ahc->init_level++;
5354
5355        /*
5356         * DMA tag for our command fifos and other data in system memory
5357         * the card's sequencer must be able to access.  For initiator
5358         * roles, we need to allocate space for the qinfifo and qoutfifo.
5359         * The qinfifo and qoutfifo are composed of 256 1 byte elements. 
5360         * When providing for the target mode role, we must additionally
5361         * provide space for the incoming target command fifo and an extra
5362         * byte to deal with a dma bug in some chip versions.
5363         */
5364        driver_data_size = 2 * 256 * sizeof(uint8_t);
5365        if ((ahc->features & AHC_TARGETMODE) != 0)
5366                driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
5367                                 + /*DMA WideOdd Bug Buffer*/1;
5368        if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5369                               /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5370                               /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5371                               /*highaddr*/BUS_SPACE_MAXADDR,
5372                               /*filter*/NULL, /*filterarg*/NULL,
5373                               driver_data_size,
5374                               /*nsegments*/1,
5375                               /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5376                               /*flags*/0, &ahc->shared_data_dmat) != 0) {
5377                return (ENOMEM);
5378        }
5379
5380        ahc->init_level++;
5381
5382        /* Allocation of driver data */
5383        if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
5384                             (void **)&ahc->qoutfifo,
5385                             BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
5386                return (ENOMEM);
5387        }
5388
5389        ahc->init_level++;
5390
5391        /* And permanently map it in */
5392        ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
5393                        ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
5394                        &ahc->shared_data_busaddr, /*flags*/0);
5395
5396        if ((ahc->features & AHC_TARGETMODE) != 0) {
5397                ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
5398                ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
5399                ahc->dma_bug_buf = ahc->shared_data_busaddr
5400                                 + driver_data_size - 1;
5401                /* All target command blocks start out invalid. */
5402                for (i = 0; i < AHC_TMODE_CMDS; i++)
5403                        ahc->targetcmds[i].cmd_valid = 0;
5404                ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
5405                ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
5406        }
5407        ahc->qinfifo = &ahc->qoutfifo[256];
5408
5409        ahc->init_level++;
5410
5411        /* Allocate SCB data now that buffer_dmat is initialized */
5412        if (ahc->scb_data->maxhscbs == 0)
5413                if (ahc_init_scbdata(ahc) != 0)
5414                        return (ENOMEM);
5415
5416        /*
5417         * Allocate a tstate to house information for our
5418         * initiator presence on the bus as well as the user
5419         * data for any target mode initiator.
5420         */
5421        if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
5422                printf("%s: unable to allocate ahc_tmode_tstate.  "
5423                       "Failing attach\n", ahc_name(ahc));
5424                return (ENOMEM);
5425        }
5426
5427        if ((ahc->features & AHC_TWIN) != 0) {
5428                if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
5429                        printf("%s: unable to allocate ahc_tmode_tstate.  "
5430                               "Failing attach\n", ahc_name(ahc));
5431                        return (ENOMEM);
5432                }
5433        }
5434
5435        if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
5436                ahc->flags |= AHC_PAGESCBS;
5437        } else {
5438                ahc->flags &= ~AHC_PAGESCBS;
5439        }
5440
5441#ifdef AHC_DEBUG
5442        if (ahc_debug & AHC_SHOW_MISC) {
5443                printf("%s: hardware scb %u bytes; kernel scb %u bytes; "
5444                       "ahc_dma %u bytes\n",
5445                        ahc_name(ahc),
5446                        (u_int)sizeof(struct hardware_scb),
5447                        (u_int)sizeof(struct scb),
5448                        (u_int)sizeof(struct ahc_dma_seg));
5449        }
5450#endif /* AHC_DEBUG */
5451
5452        /*
5453         * Look at the information that board initialization or
5454         * the board bios has left us.
5455         */
5456        if (ahc->features & AHC_TWIN) {
5457                scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5458                if ((scsi_conf & RESET_SCSI) != 0
5459                 && (ahc->flags & AHC_INITIATORROLE) != 0)
5460                        ahc->flags |= AHC_RESET_BUS_B;
5461        }
5462
5463        scsi_conf = ahc_inb(ahc, SCSICONF);
5464        if ((scsi_conf & RESET_SCSI) != 0
5465         && (ahc->flags & AHC_INITIATORROLE) != 0)
5466                ahc->flags |= AHC_RESET_BUS_A;
5467
5468        ultraenb = 0;        
5469        tagenable = ALL_TARGETS_MASK;
5470
5471        /* Grab the disconnection disable table and invert it for our needs */
5472        if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
5473                printf("%s: Host Adapter Bios disabled.  Using default SCSI "
5474                        "device parameters\n", ahc_name(ahc));
5475                ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
5476                              AHC_TERM_ENB_A|AHC_TERM_ENB_B;
5477                discenable = ALL_TARGETS_MASK;
5478                if ((ahc->features & AHC_ULTRA) != 0)
5479                        ultraenb = ALL_TARGETS_MASK;
5480        } else {
5481                discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
5482                           | ahc_inb(ahc, DISC_DSB));
5483                if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
5484                        ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
5485                                      | ahc_inb(ahc, ULTRA_ENB);
5486        }
5487
5488        if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
5489                max_targ = 7;
5490
5491        for (i = 0; i <= max_targ; i++) {
5492                struct ahc_initiator_tinfo *tinfo;
5493                struct ahc_tmode_tstate *tstate;
5494                u_int our_id;
5495                u_int target_id;
5496                char channel;
5497
5498                channel = 'A';
5499                our_id = ahc->our_id;
5500                target_id = i;
5501                if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
5502                        channel = 'B';
5503                        our_id = ahc->our_id_b;
5504                        target_id = i % 8;
5505                }
5506                tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
5507                                            target_id, &tstate);
5508                /* Default to async narrow across the board */
5509                memset(tinfo, 0, sizeof(*tinfo));
5510                if (ahc->flags & AHC_USEDEFAULTS) {
5511                        if ((ahc->features & AHC_WIDE) != 0)
5512                                tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5513
5514                        /*
5515                         * These will be truncated when we determine the
5516                         * connection type we have with the target.
5517                         */
5518                        tinfo->user.period = ahc_syncrates->period;
5519                        tinfo->user.offset = MAX_OFFSET;
5520                } else {
5521                        u_int scsirate;
5522                        uint16_t mask;
5523
5524                        /* Take the settings leftover in scratch RAM. */
5525                        scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
5526                        mask = (0x01 << i);
5527                        if ((ahc->features & AHC_ULTRA2) != 0) {
5528                                u_int offset;
5529                                u_int maxsync;
5530
5531                                if ((scsirate & SOFS) == 0x0F) {
5532                                        /*
5533                                         * Haven't negotiated yet,
5534                                         * so the format is different.
5535                                         */
5536                                        scsirate = (scsirate & SXFR) >> 4
5537                                                 | (ultraenb & mask)
5538                                                  ? 0x08 : 0x0
5539                                                 | (scsirate & WIDEXFER);
5540                                        offset = MAX_OFFSET_ULTRA2;
5541                                } else
5542                                        offset = ahc_inb(ahc, TARG_OFFSET + i);
5543                                if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
5544                                        /* Set to the lowest sync rate, 5MHz */
5545                                        scsirate |= 0x1c;
5546                                maxsync = AHC_SYNCRATE_ULTRA2;
5547                                if ((ahc->features & AHC_DT) != 0)
5548                                        maxsync = AHC_SYNCRATE_DT;
5549                                tinfo->user.period =
5550                                    ahc_find_period(ahc, scsirate, maxsync);
5551                                if (offset == 0)
5552                                        tinfo->user.period = 0;
5553                                else
5554                                        tinfo->user.offset = MAX_OFFSET;
5555                                if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
5556                                 && (ahc->features & AHC_DT) != 0)
5557                                        tinfo->user.ppr_options =
5558                                            MSG_EXT_PPR_DT_REQ;
5559                        } else if ((scsirate & SOFS) != 0) {
5560                                if ((scsirate & SXFR) == 0x40
5561                                 && (ultraenb & mask) != 0) {
5562                                        /* Treat 10MHz as a non-ultra speed */
5563                                        scsirate &= ~SXFR;
5564                                         ultraenb &= ~mask;
5565                                }
5566                                tinfo->user.period = 
5567                                    ahc_find_period(ahc, scsirate,
5568                                                    (ultraenb & mask)
5569                                                   ? AHC_SYNCRATE_ULTRA
5570                                                   : AHC_SYNCRATE_FAST);
5571                                if (tinfo->user.period != 0)
5572                                        tinfo->user.offset = MAX_OFFSET;
5573                        }
5574                        if (tinfo->user.period == 0)
5575                                tinfo->user.offset = 0;
5576                        if ((scsirate & WIDEXFER) != 0
5577                         && (ahc->features & AHC_WIDE) != 0)
5578                                tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5579                        tinfo->user.protocol_version = 4;
5580                        if ((ahc->features & AHC_DT) != 0)
5581                                tinfo->user.transport_version = 3;
5582                        else
5583                                tinfo->user.transport_version = 2;
5584                        tinfo->goal.protocol_version = 2;
5585                        tinfo->goal.transport_version = 2;
5586                        tinfo->curr.protocol_version = 2;
5587                        tinfo->curr.transport_version = 2;
5588                }
5589                tstate->ultraenb = 0;
5590        }
5591        ahc->user_discenable = discenable;
5592        ahc->user_tagenable = tagenable;
5593
5594        return (ahc->bus_chip_init(ahc));
5595}
5596
5597void
5598ahc_intr_enable(struct ahc_softc *ahc, int enable)
5599{
5600        u_int hcntrl;
5601
5602        hcntrl = ahc_inb(ahc, HCNTRL);
5603        hcntrl &= ~INTEN;
5604        ahc->pause &= ~INTEN;
5605        ahc->unpause &= ~INTEN;
5606        if (enable) {
5607                hcntrl |= INTEN;
5608                ahc->pause |= INTEN;
5609                ahc->unpause |= INTEN;
5610        }
5611        ahc_outb(ahc, HCNTRL, hcntrl);
5612}
5613
5614/*
5615 * Ensure that the card is paused in a location
5616 * outside of all critical sections and that all
5617 * pending work is completed prior to returning.
5618 * This routine should only be called from outside
5619 * an interrupt context.
5620 */
5621void
5622ahc_pause_and_flushwork(struct ahc_softc *ahc)
5623{
5624        int intstat;
5625        int maxloops;
5626        int paused;
5627
5628        maxloops = 1000;
5629        ahc->flags |= AHC_ALL_INTERRUPTS;
5630        paused = FALSE;
5631        do {
5632                if (paused) {
5633                        ahc_unpause(ahc);
5634                        /*
5635                         * Give the sequencer some time to service
5636                         * any active selections.
5637                         */
5638                        ahc_delay(500);
5639                }
5640                ahc_intr(ahc);
5641                ahc_pause(ahc);
5642                paused = TRUE;
5643                ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5644                intstat = ahc_inb(ahc, INTSTAT);
5645                if ((intstat & INT_PEND) == 0) {
5646                        ahc_clear_critical_section(ahc);
5647                        intstat = ahc_inb(ahc, INTSTAT);
5648                }
5649        } while (--maxloops
5650              && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5651              && ((intstat & INT_PEND) != 0
5652               || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5653        if (maxloops == 0) {
5654                printf("Infinite interrupt loop, INTSTAT = %x",
5655                       ahc_inb(ahc, INTSTAT));
5656        }
5657        ahc_platform_flushwork(ahc);
5658        ahc->flags &= ~AHC_ALL_INTERRUPTS;
5659}
5660
5661#ifdef CONFIG_PM
5662int
5663ahc_suspend(struct ahc_softc *ahc)
5664{
5665
5666        ahc_pause_and_flushwork(ahc);
5667
5668        if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5669                ahc_unpause(ahc);
5670                return (EBUSY);
5671        }
5672
5673#ifdef AHC_TARGET_MODE
5674        /*
5675         * XXX What about ATIOs that have not yet been serviced?
5676         * Perhaps we should just refuse to be suspended if we
5677         * are acting in a target role.
5678         */
5679        if (ahc->pending_device != NULL) {
5680                ahc_unpause(ahc);
5681                return (EBUSY);
5682        }
5683#endif
5684        ahc_shutdown(ahc);
5685        return (0);
5686}
5687
5688int
5689ahc_resume(struct ahc_softc *ahc)
5690{
5691
5692        ahc_reset(ahc, /*reinit*/TRUE);
5693        ahc_intr_enable(ahc, TRUE); 
5694        ahc_restart(ahc);
5695        return (0);
5696}
5697#endif
5698/************************** Busy Target Table *********************************/
5699/*
5700 * Return the untagged transaction id for a given target/channel lun.
5701 * Optionally, clear the entry.
5702 */
5703static u_int
5704ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5705{
5706        u_int scbid;
5707        u_int target_offset;
5708
5709        if ((ahc->flags & AHC_SCB_BTT) != 0) {
5710                u_int saved_scbptr;
5711                
5712                saved_scbptr = ahc_inb(ahc, SCBPTR);
5713                ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5714                scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5715                ahc_outb(ahc, SCBPTR, saved_scbptr);
5716        } else {
5717                target_offset = TCL_TARGET_OFFSET(tcl);
5718                scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5719        }
5720
5721        return (scbid);
5722}
5723
5724static void
5725ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5726{
5727        u_int target_offset;
5728
5729        if ((ahc->flags & AHC_SCB_BTT) != 0) {
5730                u_int saved_scbptr;
5731                
5732                saved_scbptr = ahc_inb(ahc, SCBPTR);
5733                ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5734                ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5735                ahc_outb(ahc, SCBPTR, saved_scbptr);
5736        } else {
5737                target_offset = TCL_TARGET_OFFSET(tcl);
5738                ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5739        }
5740}
5741
5742static void
5743ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5744{
5745        u_int target_offset;
5746
5747        if ((ahc->flags & AHC_SCB_BTT) != 0) {
5748                u_int saved_scbptr;
5749                
5750                saved_scbptr = ahc_inb(ahc, SCBPTR);
5751                ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5752                ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5753                ahc_outb(ahc, SCBPTR, saved_scbptr);
5754        } else {
5755                target_offset = TCL_TARGET_OFFSET(tcl);
5756                ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5757        }
5758}
5759
5760/************************** SCB and SCB queue management **********************/
5761int
5762ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5763              char channel, int lun, u_int tag, role_t role)
5764{
5765        int targ = SCB_GET_TARGET(ahc, scb);
5766        char chan = SCB_GET_CHANNEL(ahc, scb);
5767        int slun = SCB_GET_LUN(scb);
5768        int match;
5769
5770        match = ((chan == channel) || (channel == ALL_CHANNELS));
5771        if (match != 0)
5772                match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5773        if (match != 0)
5774                match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5775        if (match != 0) {
5776#ifdef AHC_TARGET_MODE
5777                int group;
5778
5779                group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5780                if (role == ROLE_INITIATOR) {
5781                        match = (group != XPT_FC_GROUP_TMODE)
5782                              && ((tag == scb->hscb->tag)
5783                               || (tag == SCB_LIST_NULL));
5784                } else if (role == ROLE_TARGET) {
5785                        match = (group == XPT_FC_GROUP_TMODE)
5786                              && ((tag == scb->io_ctx->csio.tag_id)
5787                               || (tag == SCB_LIST_NULL));
5788                }
5789#else /* !AHC_TARGET_MODE */
5790                match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5791#endif /* AHC_TARGET_MODE */
5792        }
5793
5794        return match;
5795}
5796
5797static void
5798ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5799{
5800        int        target;
5801        char        channel;
5802        int        lun;
5803
5804        target = SCB_GET_TARGET(ahc, scb);
5805        lun = SCB_GET_LUN(scb);
5806        channel = SCB_GET_CHANNEL(ahc, scb);
5807        
5808        ahc_search_qinfifo(ahc, target, channel, lun,
5809                           /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5810                           CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5811
5812        ahc_platform_freeze_devq(ahc, scb);
5813}
5814
5815void
5816ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5817{
5818        struct scb *prev_scb;
5819
5820        prev_scb = NULL;
5821        if (ahc_qinfifo_count(ahc) != 0) {
5822                u_int prev_tag;
5823                uint8_t prev_pos;
5824
5825                prev_pos = ahc->qinfifonext - 1;
5826                prev_tag = ahc->qinfifo[prev_pos];
5827                prev_scb = ahc_lookup_scb(ahc, prev_tag);
5828        }
5829        ahc_qinfifo_requeue(ahc, prev_scb, scb);
5830        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5831                ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5832        } else {
5833                ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5834        }
5835}
5836
5837static void
5838ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5839                    struct scb *scb)
5840{
5841        if (prev_scb == NULL) {
5842                ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5843        } else {
5844                prev_scb->hscb->next = scb->hscb->tag;
5845                ahc_sync_scb(ahc, prev_scb, 
5846                             BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5847        }
5848        ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5849        scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5850        ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5851}
5852
5853static int
5854ahc_qinfifo_count(struct ahc_softc *ahc)
5855{
5856        uint8_t qinpos;
5857        uint8_t diff;
5858
5859        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5860                qinpos = ahc_inb(ahc, SNSCB_QOFF);
5861                ahc_outb(ahc, SNSCB_QOFF, qinpos);
5862        } else
5863                qinpos = ahc_inb(ahc, QINPOS);
5864        diff = ahc->qinfifonext - qinpos;
5865        return (diff);
5866}
5867
5868int
5869ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5870                   int lun, u_int tag, role_t role, uint32_t status,
5871                   ahc_search_action action)
5872{
5873        struct        scb *scb;
5874        struct        scb *prev_scb;
5875        uint8_t qinstart;
5876        uint8_t qinpos;
5877        uint8_t qintail;
5878        uint8_t next;
5879        uint8_t prev;
5880        uint8_t curscbptr;
5881        int        found;
5882        int        have_qregs;
5883
5884        qintail = ahc->qinfifonext;
5885        have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5886        if (have_qregs) {
5887                qinstart = ahc_inb(ahc, SNSCB_QOFF);
5888                ahc_outb(ahc, SNSCB_QOFF, qinstart);
5889        } else
5890                qinstart = ahc_inb(ahc, QINPOS);
5891        qinpos = qinstart;
5892        found = 0;
5893        prev_scb = NULL;
5894
5895        if (action == SEARCH_COMPLETE) {
5896                /*
5897                 * Don't attempt to run any queued untagged transactions
5898                 * until we are done with the abort process.
5899                 */
5900                ahc_freeze_untagged_queues(ahc);
5901        }
5902
5903        /*
5904         * Start with an empty queue.  Entries that are not chosen
5905         * for removal will be re-added to the queue as we go.
5906         */
5907        ahc->qinfifonext = qinpos;
5908        ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5909
5910        while (qinpos != qintail) {
5911                scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5912                if (scb == NULL) {
5913                        printf("qinpos = %d, SCB index = %d\n",
5914                                qinpos, ahc->qinfifo[qinpos]);
5915                        panic("Loop 1\n");
5916                }
5917
5918                if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5919                        /*
5920                         * We found an scb that needs to be acted on.
5921                         */
5922                        found++;
5923                        switch (action) {
5924                        case SEARCH_COMPLETE:
5925                        {
5926                                cam_status ostat;
5927                                cam_status cstat;
5928
5929                                ostat = ahc_get_transaction_status(scb);
5930                                if (ostat == CAM_REQ_INPROG)
5931                                        ahc_set_transaction_status(scb, status);
5932                                cstat = ahc_get_transaction_status(scb);
5933                                if (cstat != CAM_REQ_CMP)
5934                                        ahc_freeze_scb(scb);
5935                                if ((scb->flags & SCB_ACTIVE) == 0)
5936                                        printf("Inactive SCB in qinfifo\n");
5937                                ahc_done(ahc, scb);
5938
5939                                /* FALLTHROUGH */
5940                        }
5941                        case SEARCH_REMOVE:
5942                                break;
5943                        case SEARCH_COUNT:
5944                                ahc_qinfifo_requeue(ahc, prev_scb, scb);
5945                                prev_scb = scb;
5946                                break;
5947                        }
5948                } else {
5949                        ahc_qinfifo_requeue(ahc, prev_scb, scb);
5950                        prev_scb = scb;
5951                }
5952                qinpos++;
5953        }
5954
5955        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5956                ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5957        } else {
5958                ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5959        }
5960
5961        if (action != SEARCH_COUNT
5962         && (found != 0)
5963         && (qinstart != ahc->qinfifonext)) {
5964                /*
5965                 * The sequencer may be in the process of dmaing
5966                 * down the SCB at the beginning of the queue.
5967                 * This could be problematic if either the first,
5968                 * or the second SCB is removed from the queue
5969                 * (the first SCB includes a pointer to the "next"
5970                 * SCB to dma). If we have removed any entries, swap
5971                 * the first element in the queue with the next HSCB
5972                 * so the sequencer will notice that NEXT_QUEUED_SCB
5973                 * has changed during its dma attempt and will retry
5974                 * the DMA.
5975                 */
5976                scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5977
5978                if (scb == NULL) {
5979                        printf("found = %d, qinstart = %d, qinfifionext = %d\n",
5980                                found, qinstart, ahc->qinfifonext);
5981                        panic("First/Second Qinfifo fixup\n");
5982                }
5983                /*
5984                 * ahc_swap_with_next_hscb forces our next pointer to
5985                 * point to the reserved SCB for future commands.  Save
5986                 * and restore our original next pointer to maintain
5987                 * queue integrity.
5988                 */
5989                next = scb->hscb->next;
5990                ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5991                ahc_swap_with_next_hscb(ahc, scb);
5992                scb->hscb->next = next;
5993                ahc->qinfifo[qinstart] = scb->hscb->tag;
5994
5995                /* Tell the card about the new head of the qinfifo. */
5996                ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5997
5998                /* Fixup the tail "next" pointer. */
5999                qintail = ahc->qinfifonext - 1;
6000                scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
6001                scb->hscb->next = ahc->next_queued_scb->hscb->tag;
6002        }
6003
6004        /*
6005         * Search waiting for selection list.
6006         */
6007        curscbptr = ahc_inb(ahc, SCBPTR);
6008        next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
6009        prev = SCB_LIST_NULL;
6010
6011        while (next != SCB_LIST_NULL) {
6012                uint8_t scb_index;
6013
6014                ahc_outb(ahc, SCBPTR, next);
6015                scb_index = ahc_inb(ahc, SCB_TAG);
6016                if (scb_index >= ahc->scb_data->numscbs) {
6017                        printf("Waiting List inconsistency. "
6018                               "SCB index == %d, yet numscbs == %d.",
6019                               scb_index, ahc->scb_data->numscbs);
6020                        ahc_dump_card_state(ahc);
6021                        panic("for safety");
6022                }
6023                scb = ahc_lookup_scb(ahc, scb_index);
6024                if (scb == NULL) {
6025                        printf("scb_index = %d, next = %d\n",
6026                                scb_index, next);
6027                        panic("Waiting List traversal\n");
6028                }
6029                if (ahc_match_scb(ahc, scb, target, channel,
6030                                  lun, SCB_LIST_NULL, role)) {
6031                        /*
6032                         * We found an scb that needs to be acted on.
6033                         */
6034                        found++;
6035                        switch (action) {
6036                        case SEARCH_COMPLETE:
6037                        {
6038                                cam_status ostat;
6039                                cam_status cstat;
6040
6041                                ostat = ahc_get_transaction_status(scb);
6042                                if (ostat == CAM_REQ_INPROG)
6043                                        ahc_set_transaction_status(scb,
6044                                                                   status);
6045                                cstat = ahc_get_transaction_status(scb);
6046                                if (cstat != CAM_REQ_CMP)
6047                                        ahc_freeze_scb(scb);
6048                                if ((scb->flags & SCB_ACTIVE) == 0)
6049                                        printf("Inactive SCB in Waiting List\n");
6050                                ahc_done(ahc, scb);
6051                                /* FALLTHROUGH */
6052                        }
6053                        case SEARCH_REMOVE:
6054                                next = ahc_rem_wscb(ahc, next, prev);
6055                                break;
6056                        case SEARCH_COUNT:
6057                                prev = next;
6058                                next = ahc_inb(ahc, SCB_NEXT);
6059                                break;
6060                        }
6061                } else {
6062                        
6063                        prev = next;
6064                        next = ahc_inb(ahc, SCB_NEXT);
6065                }
6066        }
6067        ahc_outb(ahc, SCBPTR, curscbptr);
6068
6069        found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
6070                                            channel, lun, status, action);
6071
6072        if (action == SEARCH_COMPLETE)
6073                ahc_release_untagged_queues(ahc);
6074        return (found);
6075}
6076
6077int
6078ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
6079                           int target, char channel, int lun, uint32_t status,
6080                           ahc_search_action action)
6081{
6082        struct        scb *scb;
6083        int        maxtarget;
6084        int        found;
6085        int        i;
6086
6087        if (action == SEARCH_COMPLETE) {
6088                /*
6089                 * Don't attempt to run any queued untagged transactions
6090                 * until we are done with the abort process.
6091                 */
6092                ahc_freeze_untagged_queues(ahc);
6093        }
6094
6095        found = 0;
6096        i = 0;
6097        if ((ahc->flags & AHC_SCB_BTT) == 0) {
6098
6099                maxtarget = 16;
6100                if (target != CAM_TARGET_WILDCARD) {
6101
6102                        i = target;
6103                        if (channel == 'B')
6104                                i += 8;
6105                        maxtarget = i + 1;
6106                }
6107        } else {
6108                maxtarget = 0;
6109        }
6110
6111        for (; i < maxtarget; i++) {
6112                struct scb_tailq *untagged_q;
6113                struct scb *next_scb;
6114
6115                untagged_q = &(ahc->untagged_queues[i]);
6116                next_scb = TAILQ_FIRST(untagged_q);
6117                while (next_scb != NULL) {
6118
6119                        scb = next_scb;
6120                        next_scb = TAILQ_NEXT(scb, links.tqe);
6121
6122                        /*
6123                         * The head of the list may be the currently
6124                         * active untagged command for a device.
6125                         * We're only searching for commands that
6126                         * have not been started.  A transaction
6127                         * marked active but still in the qinfifo
6128                         * is removed by the qinfifo scanning code
6129                         * above.
6130                         */
6131                        if ((scb->flags & SCB_ACTIVE) != 0)
6132                                continue;
6133
6134                        if (ahc_match_scb(ahc, scb, target, channel, lun,
6135                                          SCB_LIST_NULL, ROLE_INITIATOR) == 0
6136                         || (ctx != NULL && ctx != scb->io_ctx))
6137                                continue;
6138
6139                        /*
6140                         * We found an scb that needs to be acted on.
6141                         */
6142                        found++;
6143                        switch (action) {
6144                        case SEARCH_COMPLETE:
6145                        {
6146                                cam_status ostat;
6147                                cam_status cstat;
6148
6149                                ostat = ahc_get_transaction_status(scb);
6150                                if (ostat == CAM_REQ_INPROG)
6151                                        ahc_set_transaction_status(scb, status);
6152                                cstat = ahc_get_transaction_status(scb);
6153                                if (cstat != CAM_REQ_CMP)
6154                                        ahc_freeze_scb(scb);
6155                                if ((scb->flags & SCB_ACTIVE) == 0)
6156                                        printf("Inactive SCB in untaggedQ\n");
6157                                ahc_done(ahc, scb);
6158                                break;
6159                        }
6160                        case SEARCH_REMOVE:
6161                                scb->flags &= ~SCB_UNTAGGEDQ;
6162                                TAILQ_REMOVE(untagged_q, scb, links.tqe);
6163                                break;
6164                        case SEARCH_COUNT:
6165                                break;
6166                        }
6167                }
6168        }
6169
6170        if (action == SEARCH_COMPLETE)
6171                ahc_release_untagged_queues(ahc);
6172        return (found);
6173}
6174
6175int
6176ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
6177                     int lun, u_int tag, int stop_on_first, int remove,
6178                     int save_state)
6179{
6180        struct        scb *scbp;
6181        u_int        next;
6182        u_int        prev;
6183        u_int        count;
6184        u_int        active_scb;
6185
6186        count = 0;
6187        next = ahc_inb(ahc, DISCONNECTED_SCBH);
6188        prev = SCB_LIST_NULL;
6189
6190        if (save_state) {
6191                /* restore this when we're done */
6192                active_scb = ahc_inb(ahc, SCBPTR);
6193        } else
6194                /* Silence compiler */
6195                active_scb = SCB_LIST_NULL;
6196
6197        while (next != SCB_LIST_NULL) {
6198                u_int scb_index;
6199
6200                ahc_outb(ahc, SCBPTR, next);
6201                scb_index = ahc_inb(ahc, SCB_TAG);
6202                if (scb_index >= ahc->scb_data->numscbs) {
6203                        printf("Disconnected List inconsistency. "
6204                               "SCB index == %d, yet numscbs == %d.",
6205                               scb_index, ahc->scb_data->numscbs);
6206                        ahc_dump_card_state(ahc);
6207                        panic("for safety");
6208                }
6209
6210                if (next == prev) {
6211                        panic("Disconnected List Loop. "
6212                              "cur SCBPTR == %x, prev SCBPTR == %x.",
6213                              next, prev);
6214                }
6215                scbp = ahc_lookup_scb(ahc, scb_index);
6216                if (ahc_match_scb(ahc, scbp, target, channel, lun,
6217                                  tag, ROLE_INITIATOR)) {
6218                        count++;
6219                        if (remove) {
6220                                next =
6221                                    ahc_rem_scb_from_disc_list(ahc, prev, next);
6222                        } else {
6223                                prev = next;
6224                                next = ahc_inb(ahc, SCB_NEXT);
6225                        }
6226                        if (stop_on_first)
6227                                break;
6228                } else {
6229                        prev = next;
6230                        next = ahc_inb(ahc, SCB_NEXT);
6231                }
6232        }
6233        if (save_state)
6234                ahc_outb(ahc, SCBPTR, active_scb);
6235        return (count);
6236}
6237
6238/*
6239 * Remove an SCB from the on chip list of disconnected transactions.
6240 * This is empty/unused if we are not performing SCB paging.
6241 */
6242static u_int
6243ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
6244{
6245        u_int next;
6246
6247        ahc_outb(ahc, SCBPTR, scbptr);
6248        next = ahc_inb(ahc, SCB_NEXT);
6249
6250        ahc_outb(ahc, SCB_CONTROL, 0);
6251
6252        ahc_add_curscb_to_free_list(ahc);
6253
6254        if (prev != SCB_LIST_NULL) {
6255                ahc_outb(ahc, SCBPTR, prev);
6256                ahc_outb(ahc, SCB_NEXT, next);
6257        } else
6258                ahc_outb(ahc, DISCONNECTED_SCBH, next);
6259
6260        return (next);
6261}
6262
6263/*
6264 * Add the SCB as selected by SCBPTR onto the on chip list of
6265 * free hardware SCBs.  This list is empty/unused if we are not
6266 * performing SCB paging.
6267 */
6268static void
6269ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
6270{
6271        /*
6272         * Invalidate the tag so that our abort
6273         * routines don't think it's active.
6274         */
6275        ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
6276
6277        if ((ahc->flags & AHC_PAGESCBS) != 0) {
6278                ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
6279                ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
6280        }
6281}
6282
6283/*
6284 * Manipulate the waiting for selection list and return the
6285 * scb that follows the one that we remove.
6286 */
6287static u_int
6288ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
6289{
6290        u_int curscb, next;
6291
6292        /*
6293         * Select the SCB we want to abort and
6294         * pull the next pointer out of it.
6295         */
6296        curscb = ahc_inb(ahc, SCBPTR);
6297        ahc_outb(ahc, SCBPTR, scbpos);
6298        next = ahc_inb(ahc, SCB_NEXT);
6299
6300        /* Clear the necessary fields */
6301        ahc_outb(ahc, SCB_CONTROL, 0);
6302
6303        ahc_add_curscb_to_free_list(ahc);
6304
6305        /* update the waiting list */
6306        if (prev == SCB_LIST_NULL) {
6307                /* First in the list */
6308                ahc_outb(ahc, WAITING_SCBH, next); 
6309
6310                /*
6311                 * Ensure we aren't attempting to perform
6312                 * selection for this entry.
6313                 */
6314                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
6315        } else {
6316                /*
6317                 * Select the scb that pointed to us 
6318                 * and update its next pointer.
6319                 */
6320                ahc_outb(ahc, SCBPTR, prev);
6321                ahc_outb(ahc, SCB_NEXT, next);
6322        }
6323
6324        /*
6325         * Point us back at the original scb position.
6326         */
6327        ahc_outb(ahc, SCBPTR, curscb);
6328        return next;
6329}
6330
6331/******************************** Error Handling ******************************/
6332/*
6333 * Abort all SCBs that match the given description (target/channel/lun/tag),
6334 * setting their status to the passed in status if the status has not already
6335 * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
6336 * is paused before it is called.
6337 */
6338static int
6339ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
6340               int lun, u_int tag, role_t role, uint32_t status)
6341{
6342        struct        scb *scbp;
6343        struct        scb *scbp_next;
6344        u_int        active_scb;
6345        int        i, j;
6346        int        maxtarget;
6347        int        minlun;
6348        int        maxlun;
6349
6350        int        found;
6351
6352        /*
6353         * Don't attempt to run any queued untagged transactions
6354         * until we are done with the abort process.
6355         */
6356        ahc_freeze_untagged_queues(ahc);
6357
6358        /* restore this when we're done */
6359        active_scb = ahc_inb(ahc, SCBPTR);
6360
6361        found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
6362                                   role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
6363
6364        /*
6365         * Clean out the busy target table for any untagged commands.
6366         */
6367        i = 0;
6368        maxtarget = 16;
6369        if (target != CAM_TARGET_WILDCARD) {
6370                i = target;
6371                if (channel == 'B')
6372                        i += 8;
6373                maxtarget = i + 1;
6374        }
6375
6376        if (lun == CAM_LUN_WILDCARD) {
6377
6378                /*
6379                 * Unless we are using an SCB based
6380                 * busy targets table, there is only
6381                 * one table entry for all luns of
6382                 * a target.
6383                 */
6384                minlun = 0;
6385                maxlun = 1;
6386                if ((ahc->flags & AHC_SCB_BTT) != 0)
6387                        maxlun = AHC_NUM_LUNS;
6388        } else {
6389                minlun = lun;
6390                maxlun = lun + 1;
6391        }
6392
6393        if (role != ROLE_TARGET) {
6394                for (;i < maxtarget; i++) {
6395                        for (j = minlun;j < maxlun; j++) {
6396                                u_int scbid;
6397                                u_int tcl;
6398
6399                                tcl = BUILD_TCL(i << 4, j);
6400                                scbid = ahc_index_busy_tcl(ahc, tcl);
6401                                scbp = ahc_lookup_scb(ahc, scbid);
6402                                if (scbp == NULL
6403                                 || ahc_match_scb(ahc, scbp, target, channel,
6404                                                  lun, tag, role) == 0)
6405                                        continue;
6406                                ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
6407                        }
6408                }
6409
6410                /*
6411                 * Go through the disconnected list and remove any entries we
6412                 * have queued for completion, 0'ing their control byte too.
6413                 * We save the active SCB and restore it ourselves, so there
6414                 * is no reason for this search to restore it too.
6415                 */
6416                ahc_search_disc_list(ahc, target, channel, lun, tag,
6417                                     /*stop_on_first*/FALSE, /*remove*/TRUE,
6418                                     /*save_state*/FALSE);
6419        }
6420
6421        /*
6422         * Go through the hardware SCB array looking for commands that
6423         * were active but not on any list.  In some cases, these remnants
6424         * might not still have mappings in the scbindex array (e.g. unexpected
6425         * bus free with the same scb queued for an abort).  Don't hold this
6426         * against them.
6427         */
6428        for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6429                u_int scbid;
6430
6431                ahc_outb(ahc, SCBPTR, i);
6432                scbid = ahc_inb(ahc, SCB_TAG);
6433                scbp = ahc_lookup_scb(ahc, scbid);
6434                if ((scbp == NULL && scbid != SCB_LIST_NULL)
6435                 || (scbp != NULL
6436                  && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
6437                        ahc_add_curscb_to_free_list(ahc);
6438        }
6439
6440        /*
6441         * Go through the pending CCB list and look for
6442         * commands for this target that are still active.
6443         * These are other tagged commands that were
6444         * disconnected when the reset occurred.
6445         */
6446        scbp_next = LIST_FIRST(&ahc->pending_scbs);
6447        while (scbp_next != NULL) {
6448                scbp = scbp_next;
6449                scbp_next = LIST_NEXT(scbp, pending_links);
6450                if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
6451                        cam_status ostat;
6452
6453                        ostat = ahc_get_transaction_status(scbp);
6454                        if (ostat == CAM_REQ_INPROG)
6455                                ahc_set_transaction_status(scbp, status);
6456                        if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
6457                                ahc_freeze_scb(scbp);
6458                        if ((scbp->flags & SCB_ACTIVE) == 0)
6459                                printf("Inactive SCB on pending list\n");
6460                        ahc_done(ahc, scbp);
6461                        found++;
6462                }
6463        }
6464        ahc_outb(ahc, SCBPTR, active_scb);
6465        ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
6466        ahc_release_untagged_queues(ahc);
6467        return found;
6468}
6469
6470static void
6471ahc_reset_current_bus(struct ahc_softc *ahc)
6472{
6473        uint8_t scsiseq;
6474
6475        ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
6476        scsiseq = ahc_inb(ahc, SCSISEQ);
6477        ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
6478        ahc_flush_device_writes(ahc);
6479        ahc_delay(AHC_BUSRESET_DELAY);
6480        /* Turn off the bus reset */
6481        ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
6482
6483        ahc_clear_intstat(ahc);
6484
6485        /* Re-enable reset interrupts */
6486        ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
6487}
6488
6489int
6490ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
6491{
6492        struct        ahc_devinfo devinfo;
6493        u_int        initiator, target, max_scsiid;
6494        u_int        sblkctl;
6495        u_int        scsiseq;
6496        u_int        simode1;
6497        int        found;
6498        int        restart_needed;
6499        char        cur_channel;
6500
6501        ahc->pending_device = NULL;
6502
6503        ahc_compile_devinfo(&devinfo,
6504                            CAM_TARGET_WILDCARD,
6505                            CAM_TARGET_WILDCARD,
6506                            CAM_LUN_WILDCARD,
6507                            channel, ROLE_UNKNOWN);
6508        ahc_pause(ahc);
6509
6510        /* Make sure the sequencer is in a safe location. */
6511        ahc_clear_critical_section(ahc);
6512
6513        /*
6514         * Run our command complete fifos to ensure that we perform
6515         * completion processing on any commands that 'completed'
6516         * before the reset occurred.
6517         */
6518        ahc_run_qoutfifo(ahc);
6519#ifdef AHC_TARGET_MODE
6520        /*
6521         * XXX - In Twin mode, the tqinfifo may have commands
6522         *         for an unaffected channel in it.  However, if
6523         *         we have run out of ATIO resources to drain that
6524         *         queue, we may not get them all out here.  Further,
6525         *         the blocked transactions for the reset channel
6526         *         should just be killed off, irrespecitve of whether
6527         *         we are blocked on ATIO resources.  Write a routine
6528         *         to compact the tqinfifo appropriately.
6529         */
6530        if ((ahc->flags & AHC_TARGETROLE) != 0) {
6531                ahc_run_tqinfifo(ahc, /*paused*/TRUE);
6532        }
6533#endif
6534
6535        /*
6536         * Reset the bus if we are initiating this reset
6537         */
6538        sblkctl = ahc_inb(ahc, SBLKCTL);
6539        cur_channel = 'A';
6540        if ((ahc->features & AHC_TWIN) != 0
6541         && ((sblkctl & SELBUSB) != 0))
6542            cur_channel = 'B';
6543        scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
6544        if (cur_channel != channel) {
6545                /* Case 1: Command for another bus is active
6546                 * Stealthily reset the other bus without
6547                 * upsetting the current bus.
6548                 */
6549                ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
6550                simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6551#ifdef AHC_TARGET_MODE
6552                /*
6553                 * Bus resets clear ENSELI, so we cannot
6554                 * defer re-enabling bus reset interrupts
6555                 * if we are in target mode.
6556                 */
6557                if ((ahc->flags & AHC_TARGETROLE) != 0)
6558                        simode1 |= ENSCSIRST;
6559#endif
6560                ahc_outb(ahc, SIMODE1, simode1);
6561                if (initiate_reset)
6562                        ahc_reset_current_bus(ahc);
6563                ahc_clear_intstat(ahc);
6564                ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6565                ahc_outb(ahc, SBLKCTL, sblkctl);
6566                restart_needed = FALSE;
6567        } else {
6568                /* Case 2: A command from this bus is active or we're idle */
6569                simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6570#ifdef AHC_TARGET_MODE
6571                /*
6572                 * Bus resets clear ENSELI, so we cannot
6573                 * defer re-enabling bus reset interrupts
6574                 * if we are in target mode.
6575                 */
6576                if ((ahc->flags & AHC_TARGETROLE) != 0)
6577                        simode1 |= ENSCSIRST;
6578#endif
6579                ahc_outb(ahc, SIMODE1, simode1);
6580                if (initiate_reset)
6581                        ahc_reset_current_bus(ahc);
6582                ahc_clear_intstat(ahc);
6583                ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6584                restart_needed = TRUE;
6585        }
6586
6587        /*
6588         * Clean up all the state information for the
6589         * pending transactions on this bus.
6590         */
6591        found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6592                               CAM_LUN_WILDCARD, SCB_LIST_NULL,
6593                               ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6594
6595        max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6596
6597#ifdef AHC_TARGET_MODE
6598        /*
6599         * Send an immediate notify ccb to all target more peripheral
6600         * drivers affected by this action.
6601         */
6602        for (target = 0; target <= max_scsiid; target++) {
6603                struct ahc_tmode_tstate* tstate;
6604                u_int lun;
6605
6606                tstate = ahc->enabled_targets[target];
6607                if (tstate == NULL)
6608                        continue;
6609                for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6610                        struct ahc_tmode_lstate* lstate;
6611
6612                        lstate = tstate->enabled_luns[lun];
6613                        if (lstate == NULL)
6614                                continue;
6615
6616                        ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6617                                               EVENT_TYPE_BUS_RESET, /*arg*/0);
6618                        ahc_send_lstate_events(ahc, lstate);
6619                }
6620        }
6621#endif
6622        /* Notify the XPT that a bus reset occurred */
6623        ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6624                       CAM_LUN_WILDCARD, AC_BUS_RESET);
6625
6626        /*
6627         * Revert to async/narrow transfers until we renegotiate.
6628         */
6629        for (target = 0; target <= max_scsiid; target++) {
6630
6631                if (ahc->enabled_targets[target] == NULL)
6632                        continue;
6633                for (initiator = 0; initiator <= max_scsiid; initiator++) {
6634                        struct ahc_devinfo devinfo;
6635
6636                        ahc_compile_devinfo(&devinfo, target, initiator,
6637                                            CAM_LUN_WILDCARD,
6638                                            channel, ROLE_UNKNOWN);
6639                        ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6640                                      AHC_TRANS_CUR, /*paused*/TRUE);
6641                        ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6642                                         /*period*/0, /*offset*/0,
6643                                         /*ppr_options*/0, AHC_TRANS_CUR,
6644                                         /*paused*/TRUE);
6645                }
6646        }
6647
6648        if (restart_needed)
6649                ahc_restart(ahc);
6650        else
6651                ahc_unpause(ahc);
6652        return found;
6653}
6654
6655
6656/***************************** Residual Processing ****************************/
6657/*
6658 * Calculate the residual for a just completed SCB.
6659 */
6660static void
6661ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6662{
6663        struct hardware_scb *hscb;
6664        struct status_pkt *spkt;
6665        uint32_t sgptr;
6666        uint32_t resid_sgptr;
6667        uint32_t resid;
6668
6669        /*
6670         * 5 cases.
6671         * 1) No residual.
6672         *    SG_RESID_VALID clear in sgptr.
6673         * 2) Transferless command
6674         * 3) Never performed any transfers.
6675         *    sgptr has SG_FULL_RESID set.
6676         * 4) No residual but target did not
6677         *    save data pointers after the
6678         *    last transfer, so sgptr was
6679         *    never updated.
6680         * 5) We have a partial residual.
6681         *    Use residual_sgptr to determine
6682         *    where we are.
6683         */
6684
6685        hscb = scb->hscb;
6686        sgptr = ahc_le32toh(hscb->sgptr);
6687        if ((sgptr & SG_RESID_VALID) == 0)
6688                /* Case 1 */
6689                return;
6690        sgptr &= ~SG_RESID_VALID;
6691
6692        if ((sgptr & SG_LIST_NULL) != 0)
6693                /* Case 2 */
6694                return;
6695
6696        spkt = &hscb->shared_data.status;
6697        resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6698        if ((sgptr & SG_FULL_RESID) != 0) {
6699                /* Case 3 */
6700                resid = ahc_get_transfer_length(scb);
6701        } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6702                /* Case 4 */
6703                return;
6704        } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6705                panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6706        } else {
6707                struct ahc_dma_seg *sg;
6708
6709                /*
6710                 * Remainder of the SG where the transfer
6711                 * stopped.  
6712                 */
6713                resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6714                sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6715
6716                /* The residual sg_ptr always points to the next sg */
6717                sg--;
6718
6719                /*
6720                 * Add up the contents of all residual
6721                 * SG segments that are after the SG where
6722                 * the transfer stopped.
6723                 */
6724                while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6725                        sg++;
6726                        resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6727                }
6728        }
6729        if ((scb->flags & SCB_SENSE) == 0)
6730                ahc_set_residual(scb, resid);
6731        else
6732                ahc_set_sense_residual(scb, resid);
6733
6734#ifdef AHC_DEBUG
6735        if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6736                ahc_print_path(ahc, scb);
6737                printf("Handled %sResidual of %d bytes\n",
6738                       (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6739        }
6740#endif
6741}
6742
6743/******************************* Target Mode **********************************/
6744#ifdef AHC_TARGET_MODE
6745/*
6746 * Add a target mode event to this lun's queue
6747 */
6748static void
6749ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6750                       u_int initiator_id, u_int event_type, u_int event_arg)
6751{
6752        struct ahc_tmode_event *event;
6753        int pending;
6754
6755        xpt_freeze_devq(lstate->path, /*count*/1);
6756        if (lstate->event_w_idx >= lstate->event_r_idx)
6757                pending = lstate->event_w_idx - lstate->event_r_idx;
6758        else
6759                pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6760                        - (lstate->event_r_idx - lstate->event_w_idx);
6761
6762        if (event_type == EVENT_TYPE_BUS_RESET
6763         || event_type == MSG_BUS_DEV_RESET) {
6764                /*
6765                 * Any earlier events are irrelevant, so reset our buffer.
6766                 * This has the effect of allowing us to deal with reset
6767                 * floods (an external device holding down the reset line)
6768                 * without losing the event that is really interesting.
6769                 */
6770                lstate->event_r_idx = 0;
6771                lstate->event_w_idx = 0;
6772                xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6773        }
6774
6775        if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6776                xpt_print_path(lstate->path);
6777                printf("immediate event %x:%x lost\n",
6778                       lstate->event_buffer[lstate->event_r_idx].event_type,
6779                       lstate->event_buffer[lstate->event_r_idx].event_arg);
6780                lstate->event_r_idx++;
6781                if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6782                        lstate->event_r_idx = 0;
6783                xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6784        }
6785
6786        event = &lstate->event_buffer[lstate->event_w_idx];
6787        event->initiator_id = initiator_id;
6788        event->event_type = event_type;
6789        event->event_arg = event_arg;
6790        lstate->event_w_idx++;
6791        if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6792                lstate->event_w_idx = 0;
6793}
6794
6795/*
6796 * Send any target mode events queued up waiting
6797 * for immediate notify resources.
6798 */
6799void
6800ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6801{
6802        struct ccb_hdr *ccbh;
6803        struct ccb_immed_notify *inot;
6804
6805        while (lstate->event_r_idx != lstate->event_w_idx
6806            && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6807                struct ahc_tmode_event *event;
6808
6809                event = &lstate->event_buffer[lstate->event_r_idx];
6810                SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6811                inot = (struct ccb_immed_notify *)ccbh;
6812                switch (event->event_type) {
6813                case EVENT_TYPE_BUS_RESET:
6814                        ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6815                        break;
6816                default:
6817                        ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6818                        inot->message_args[0] = event->event_type;
6819                        inot->message_args[1] = event->event_arg;
6820                        break;
6821                }
6822                inot->initiator_id = event->initiator_id;
6823                inot->sense_len = 0;
6824                xpt_done((union ccb *)inot);
6825                lstate->event_r_idx++;
6826                if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6827                        lstate->event_r_idx = 0;
6828        }
6829}
6830#endif
6831
6832/******************** Sequencer Program Patching/Download *********************/
6833
6834#ifdef AHC_DUMP_SEQ
6835void
6836ahc_dumpseq(struct ahc_softc* ahc)
6837{
6838        int i;
6839
6840        ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6841        ahc_outb(ahc, SEQADDR0, 0);
6842        ahc_outb(ahc, SEQADDR1, 0);
6843        for (i = 0; i < ahc->instruction_ram_size; i++) {
6844                uint8_t ins_bytes[4];
6845
6846                ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6847                printf("0x%08x\n", ins_bytes[0] << 24
6848                                 | ins_bytes[1] << 16
6849                                 | ins_bytes[2] << 8
6850                                 | ins_bytes[3]);
6851        }
6852}
6853#endif
6854
6855static int
6856ahc_loadseq(struct ahc_softc *ahc)
6857{
6858        struct        cs cs_table[num_critical_sections];
6859        u_int        begin_set[num_critical_sections];
6860        u_int        end_set[num_critical_sections];
6861        const struct patch *cur_patch;
6862        u_int        cs_count;
6863        u_int        cur_cs;
6864        u_int        i;
6865        u_int        skip_addr;
6866        u_int        sg_prefetch_cnt;
6867        int        downloaded;
6868        uint8_t        download_consts[7];
6869
6870        /*
6871         * Start out with 0 critical sections
6872         * that apply to this firmware load.
6873         */
6874        cs_count = 0;
6875        cur_cs = 0;
6876        memset(begin_set, 0, sizeof(begin_set));
6877        memset(end_set, 0, sizeof(end_set));
6878
6879        /* Setup downloadable constant table */
6880        download_consts[QOUTFIFO_OFFSET] = 0;
6881        if (ahc->targetcmds != NULL)
6882                download_consts[QOUTFIFO_OFFSET] += 32;
6883        download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6884        download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6885        download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6886        sg_prefetch_cnt = ahc->pci_cachesize;
6887        if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6888                sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6889        download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6890        download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6891        download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6892
6893        cur_patch = patches;
6894        downloaded = 0;
6895        skip_addr = 0;
6896        ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6897        ahc_outb(ahc, SEQADDR0, 0);
6898        ahc_outb(ahc, SEQADDR1, 0);
6899
6900        for (i = 0; i < sizeof(seqprog)/4; i++) {
6901                if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6902                        /*
6903                         * Don't download this instruction as it
6904                         * is in a patch that was removed.
6905                         */
6906                        continue;
6907                }
6908
6909                if (downloaded == ahc->instruction_ram_size) {
6910                        /*
6911                         * We're about to exceed the instruction
6912                         * storage capacity for this chip.  Fail
6913                         * the load.
6914                         */
6915                        printf("\n%s: Program too large for instruction memory "
6916                               "size of %d!\n", ahc_name(ahc),
6917                               ahc->instruction_ram_size);
6918                        return (ENOMEM);
6919                }
6920
6921                /*
6922                 * Move through the CS table until we find a CS
6923                 * that might apply to this instruction.
6924                 */
6925                for (; cur_cs < num_critical_sections; cur_cs++) {
6926                        if (critical_sections[cur_cs].end <= i) {
6927                                if (begin_set[cs_count] == TRUE
6928                                 && end_set[cs_count] == FALSE) {
6929                                        cs_table[cs_count].end = downloaded;
6930                                         end_set[cs_count] = TRUE;
6931                                        cs_count++;
6932                                }
6933                                continue;
6934                        }
6935                        if (critical_sections[cur_cs].begin <= i
6936                         && begin_set[cs_count] == FALSE) {
6937                                cs_table[cs_count].begin = downloaded;
6938                                begin_set[cs_count] = TRUE;
6939                        }
6940                        break;
6941                }
6942                ahc_download_instr(ahc, i, download_consts);
6943                downloaded++;
6944        }
6945
6946        ahc->num_critical_sections = cs_count;
6947        if (cs_count != 0) {
6948
6949                cs_count *= sizeof(struct cs);
6950                ahc->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
6951                if (ahc->critical_sections == NULL)
6952                        panic("ahc_loadseq: Could not malloc");
6953                memcpy(ahc->critical_sections, cs_table, cs_count);
6954        }
6955        ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6956
6957        if (bootverbose) {
6958                printf(" %d instructions downloaded\n", downloaded);
6959                printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6960                       ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6961        }
6962        return (0);
6963}
6964
6965static int
6966ahc_check_patch(struct ahc_softc *ahc, const struct patch **start_patch,
6967                u_int start_instr, u_int *skip_addr)
6968{
6969        const struct patch *cur_patch;
6970        const struct patch *last_patch;
6971        u_int        num_patches;
6972
6973        num_patches = ARRAY_SIZE(patches);
6974        last_patch = &patches[num_patches];
6975        cur_patch = *start_patch;
6976
6977        while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6978
6979                if (cur_patch->patch_func(ahc) == 0) {
6980
6981                        /* Start rejecting code */
6982                        *skip_addr = start_instr + cur_patch->skip_instr;
6983                        cur_patch += cur_patch->skip_patch;
6984                } else {
6985                        /* Accepted this patch.  Advance to the next
6986                         * one and wait for our intruction pointer to
6987                         * hit this point.
6988                         */
6989                        cur_patch++;
6990                }
6991        }
6992
6993        *start_patch = cur_patch;
6994        if (start_instr < *skip_addr)
6995                /* Still skipping */
6996                return (0);
6997
6998        return (1);
6999}
7000
7001static void
7002ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
7003{
7004        union        ins_formats instr;
7005        struct        ins_format1 *fmt1_ins;
7006        struct        ins_format3 *fmt3_ins;
7007        u_int        opcode;
7008
7009        /*
7010         * The firmware is always compiled into a little endian format.
7011         */
7012        instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
7013
7014        fmt1_ins = &instr.format1;
7015        fmt3_ins = NULL;
7016
7017        /* Pull the opcode */
7018        opcode = instr.format1.opcode;
7019        switch (opcode) {
7020        case AIC_OP_JMP:
7021        case AIC_OP_JC:
7022        case AIC_OP_JNC:
7023        case AIC_OP_CALL:
7024        case AIC_OP_JNE:
7025        case AIC_OP_JNZ:
7026        case AIC_OP_JE:
7027        case AIC_OP_JZ:
7028        {
7029                const struct patch *cur_patch;
7030                int address_offset;
7031                u_int address;
7032                u_int skip_addr;
7033                u_int i;
7034
7035                fmt3_ins = &instr.format3;
7036                address_offset = 0;
7037                address = fmt3_ins->address;
7038                cur_patch = patches;
7039                skip_addr = 0;
7040
7041                for (i = 0; i < address;) {
7042
7043                        ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
7044
7045                        if (skip_addr > i) {
7046                                int end_addr;
7047
7048                                end_addr = min(address, skip_addr);
7049                                address_offset += end_addr - i;
7050                                i = skip_addr;
7051                        } else {
7052                                i++;
7053                        }
7054                }
7055                address -= address_offset;
7056                fmt3_ins->address = address;
7057                /* FALLTHROUGH */
7058        }
7059        case AIC_OP_OR:
7060        case AIC_OP_AND:
7061        case AIC_OP_XOR:
7062        case AIC_OP_ADD:
7063        case AIC_OP_ADC:
7064        case AIC_OP_BMOV:
7065                if (fmt1_ins->parity != 0) {
7066                        fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
7067                }
7068                fmt1_ins->parity = 0;
7069                if ((ahc->features & AHC_CMD_CHAN) == 0
7070                 && opcode == AIC_OP_BMOV) {
7071                        /*
7072                         * Block move was added at the same time
7073                         * as the command channel.  Verify that
7074                         * this is only a move of a single element
7075                         * and convert the BMOV to a MOV
7076                         * (AND with an immediate of FF).
7077                         */
7078                        if (fmt1_ins->immediate != 1)
7079                                panic("%s: BMOV not supported\n",
7080                                      ahc_name(ahc));
7081                        fmt1_ins->opcode = AIC_OP_AND;
7082                        fmt1_ins->immediate = 0xff;
7083                }
7084                /* FALLTHROUGH */
7085        case AIC_OP_ROL:
7086                if ((ahc->features & AHC_ULTRA2) != 0) {
7087                        int i, count;
7088
7089                        /* Calculate odd parity for the instruction */
7090                        for (i = 0, count = 0; i < 31; i++) {
7091                                uint32_t mask;
7092
7093                                mask = 0x01 << i;
7094                                if ((instr.integer & mask) != 0)
7095                                        count++;
7096                        }
7097                        if ((count & 0x01) == 0)
7098                                instr.format1.parity = 1;
7099                } else {
7100                        /* Compress the instruction for older sequencers */
7101                        if (fmt3_ins != NULL) {
7102                                instr.integer =
7103                                        fmt3_ins->immediate
7104                                      | (fmt3_ins->source << 8)
7105                                      | (fmt3_ins->address << 16)
7106                                      |        (fmt3_ins->opcode << 25);
7107                        } else {
7108                                instr.integer =
7109                                        fmt1_ins->immediate
7110                                      | (fmt1_ins->source << 8)
7111                                      | (fmt1_ins->destination << 16)
7112                                      |        (fmt1_ins->ret << 24)
7113                                      |        (fmt1_ins->opcode << 25);
7114                        }
7115                }
7116                /* The sequencer is a little endian cpu */
7117                instr.integer = ahc_htole32(instr.integer);
7118                ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
7119                break;
7120        default:
7121                panic("Unknown opcode encountered in seq program");
7122                break;
7123        }
7124}
7125
7126int
7127ahc_print_register(const ahc_reg_parse_entry_t *table, u_int num_entries,
7128                   const char *name, u_int address, u_int value,
7129                   u_int *cur_column, u_int wrap_point)
7130{
7131        int        printed;
7132        u_int        printed_mask;
7133
7134        if (cur_column != NULL && *cur_column >= wrap_point) {
7135                printf("\n");
7136                *cur_column = 0;
7137        }
7138        printed = printf("%s[0x%x]", name, value);
7139        if (table == NULL) {
7140                printed += printf(" ");
7141                *cur_column += printed;
7142                return (printed);
7143        }
7144        printed_mask = 0;
7145        while (printed_mask != 0xFF) {
7146                int entry;
7147
7148                for (entry = 0; entry < num_entries; entry++) {
7149                        if (((value & table[entry].mask)
7150                          != table[entry].value)
7151                         || ((printed_mask & table[entry].mask)
7152                          == table[entry].mask))
7153                                continue;
7154
7155                        printed += printf("%s%s",
7156                                          printed_mask == 0 ? ":(" : "|",
7157                                          table[entry].name);
7158                        printed_mask |= table[entry].mask;
7159                        
7160                        break;
7161                }
7162                if (entry >= num_entries)
7163                        break;
7164        }
7165        if (printed_mask != 0)
7166                printed += printf(") ");
7167        else
7168                printed += printf(" ");
7169        if (cur_column != NULL)
7170                *cur_column += printed;
7171        return (printed);
7172}
7173
7174void
7175ahc_dump_card_state(struct ahc_softc *ahc)
7176{
7177        struct        scb *scb;
7178        struct        scb_tailq *untagged_q;
7179        u_int        cur_col;
7180        int        paused;
7181        int        target;
7182        int        maxtarget;
7183        int        i;
7184        uint8_t last_phase;
7185        uint8_t qinpos;
7186        uint8_t qintail;
7187        uint8_t qoutpos;
7188        uint8_t scb_index;
7189        uint8_t saved_scbptr;
7190
7191        if (ahc_is_paused(ahc)) {
7192                paused = 1;
7193        } else {
7194                paused = 0;
7195                ahc_pause(ahc);
7196        }
7197
7198        saved_scbptr = ahc_inb(ahc, SCBPTR);
7199        last_phase = ahc_inb(ahc, LASTPHASE);
7200        printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
7201               "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
7202               ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
7203               ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
7204        if (paused)
7205                printf("Card was paused\n");
7206        printf("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
7207               ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
7208               ahc_inb(ahc, ARG_2));
7209        printf("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
7210               ahc_inb(ahc, SCBPTR));
7211        cur_col = 0;
7212        if ((ahc->features & AHC_DT) != 0)
7213                ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
7214        ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
7215        ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
7216        ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
7217        ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
7218        ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
7219        ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
7220        ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
7221        ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
7222        ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
7223        ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
7224        ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
7225        ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
7226        ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
7227        ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
7228        ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
7229        ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
7230        ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
7231        ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
7232        if (cur_col != 0)
7233                printf("\n");
7234        printf("STACK:");
7235        for (i = 0; i < STACK_SIZE; i++)
7236               printf(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
7237        printf("\nSCB count = %d\n", ahc->scb_data->numscbs);
7238        printf("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
7239        printf("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
7240        /* QINFIFO */
7241        printf("QINFIFO entries: ");
7242        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
7243                qinpos = ahc_inb(ahc, SNSCB_QOFF);
7244                ahc_outb(ahc, SNSCB_QOFF, qinpos);
7245        } else
7246                qinpos = ahc_inb(ahc, QINPOS);
7247        qintail = ahc->qinfifonext;
7248        while (qinpos != qintail) {
7249                printf("%d ", ahc->qinfifo[qinpos]);
7250                qinpos++;
7251        }
7252        printf("\n");
7253
7254        printf("Waiting Queue entries: ");
7255        scb_index = ahc_inb(ahc, WAITING_SCBH);
7256        i = 0;
7257        while (scb_index != SCB_LIST_NULL && i++ < 256) {
7258                ahc_outb(ahc, SCBPTR, scb_index);
7259                printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7260                scb_index = ahc_inb(ahc, SCB_NEXT);
7261        }
7262        printf("\n");
7263
7264        printf("Disconnected Queue entries: ");
7265        scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
7266        i = 0;
7267        while (scb_index != SCB_LIST_NULL && i++ < 256) {
7268                ahc_outb(ahc, SCBPTR, scb_index);
7269                printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7270                scb_index = ahc_inb(ahc, SCB_NEXT);
7271        }
7272        printf("\n");
7273                
7274        ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
7275        printf("QOUTFIFO entries: ");
7276        qoutpos = ahc->qoutfifonext;
7277        i = 0;
7278        while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
7279                printf("%d ", ahc->qoutfifo[qoutpos]);
7280                qoutpos++;
7281        }
7282        printf("\n");
7283
7284        printf("Sequencer Free SCB List: ");
7285        scb_index = ahc_inb(ahc, FREE_SCBH);
7286        i = 0;
7287        while (scb_index != SCB_LIST_NULL && i++ < 256) {
7288                ahc_outb(ahc, SCBPTR, scb_index);
7289                printf("%d ", scb_index);
7290                scb_index = ahc_inb(ahc, SCB_NEXT);
7291        }
7292        printf("\n");
7293
7294        printf("Sequencer SCB Info: ");
7295        for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
7296                ahc_outb(ahc, SCBPTR, i);
7297                cur_col = printf("\n%3d ", i);
7298
7299                ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
7300                ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
7301                ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
7302                ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7303        }
7304        printf("\n");
7305
7306        printf("Pending list: ");
7307        i = 0;
7308        LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7309                if (i++ > 256)
7310                        break;
7311                cur_col = printf("\n%3d ", scb->hscb->tag);
7312                ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
7313                ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
7314                ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
7315                if ((ahc->flags & AHC_PAGESCBS) == 0) {
7316                        ahc_outb(ahc, SCBPTR, scb->hscb->tag);
7317                        printf("(");
7318                        ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
7319                                              &cur_col, 60);
7320                        ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7321                        printf(")");
7322                }
7323        }
7324        printf("\n");
7325
7326        printf("Kernel Free SCB list: ");
7327        i = 0;
7328        SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
7329                if (i++ > 256)
7330                        break;
7331                printf("%d ", scb->hscb->tag);
7332        }
7333        printf("\n");
7334
7335        maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
7336        for (target = 0; target <= maxtarget; target++) {
7337                untagged_q = &ahc->untagged_queues[target];
7338                if (TAILQ_FIRST(untagged_q) == NULL)
7339                        continue;
7340                printf("Untagged Q(%d): ", target);
7341                i = 0;
7342                TAILQ_FOREACH(scb, untagged_q, links.tqe) {
7343                        if (i++ > 256)
7344                                break;
7345                        printf("%d ", scb->hscb->tag);
7346                }
7347                printf("\n");
7348        }
7349
7350        ahc_platform_dump_card_state(ahc);
7351        printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
7352        ahc_outb(ahc, SCBPTR, saved_scbptr);
7353        if (paused == 0)
7354                ahc_unpause(ahc);
7355}
7356
7357/************************* Target Mode ****************************************/
7358#ifdef AHC_TARGET_MODE
7359cam_status
7360ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
7361                    struct ahc_tmode_tstate **tstate,
7362                    struct ahc_tmode_lstate **lstate,
7363                    int notfound_failure)
7364{
7365
7366        if ((ahc->features & AHC_TARGETMODE) == 0)
7367                return (CAM_REQ_INVALID);
7368
7369        /*
7370         * Handle the 'black hole' device that sucks up
7371         * requests to unattached luns on enabled targets.
7372         */
7373        if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
7374         && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
7375                *tstate = NULL;
7376                *lstate = ahc->black_hole;
7377        } else {
7378                u_int max_id;
7379
7380                max_id = (ahc->features & AHC_WIDE) ? 16 : 8;
7381                if (ccb->ccb_h.target_id >= max_id)
7382                        return (CAM_TID_INVALID);
7383
7384                if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
7385                        return (CAM_LUN_INVALID);
7386
7387                *tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
7388                *lstate = NULL;
7389                if (*tstate != NULL)
7390                        *lstate =
7391                            (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
7392        }
7393
7394        if (notfound_failure != 0 && *lstate == NULL)
7395                return (CAM_PATH_INVALID);
7396
7397        return (CAM_REQ_CMP);
7398}
7399
7400void
7401ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
7402{
7403        struct           ahc_tmode_tstate *tstate;
7404        struct           ahc_tmode_lstate *lstate;
7405        struct           ccb_en_lun *cel;
7406        cam_status status;
7407        u_long           s;
7408        u_int           target;
7409        u_int           lun;
7410        u_int           target_mask;
7411        u_int           our_id;
7412        int           error;
7413        char           channel;
7414
7415        status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
7416                                     /*notfound_failure*/FALSE);
7417
7418        if (status != CAM_REQ_CMP) {
7419                ccb->ccb_h.status = status;
7420                return;
7421        }
7422
7423        if (cam_sim_bus(sim) == 0)
7424                our_id = ahc->our_id;
7425        else
7426                our_id = ahc->our_id_b;
7427
7428        if (ccb->ccb_h.target_id != our_id) {
7429                /*
7430                 * our_id represents our initiator ID, or
7431                 * the ID of the first target to have an
7432                 * enabled lun in target mode.  There are
7433                 * two cases that may preclude enabling a
7434                 * target id other than our_id.
7435                 *
7436                 *   o our_id is for an active initiator role.
7437                 *     Since the hardware does not support
7438                 *     reselections to the initiator role at
7439                 *     anything other than our_id, and our_id
7440                 *     is used by the hardware to indicate the
7441                 *     ID to use for both select-out and
7442                 *     reselect-out operations, the only target
7443                 *     ID we can support in this mode is our_id.
7444                 *
7445                 *   o The MULTARGID feature is not available and
7446                 *     a previous target mode ID has been enabled.
7447                 */
7448                if ((ahc->features & AHC_MULTIROLE) != 0) {
7449
7450                        if ((ahc->features & AHC_MULTI_TID) != 0
7451                            && (ahc->flags & AHC_INITIATORROLE) != 0) {
7452                                /*
7453                                 * Only allow additional targets if
7454                                 * the initiator role is disabled.
7455                                 * The hardware cannot handle a re-select-in
7456                                 * on the initiator id during a re-select-out
7457                                 * on a different target id.
7458                                 */
7459                                status = CAM_TID_INVALID;
7460                        } else if ((ahc->flags & AHC_INITIATORROLE) != 0
7461                                || ahc->enabled_luns > 0) {
7462                                /*
7463                                 * Only allow our target id to change
7464                                 * if the initiator role is not configured
7465                                 * and there are no enabled luns which
7466                                 * are attached to the currently registered
7467                                 * scsi id.
7468                                 */
7469                                status = CAM_TID_INVALID;