root/dev/raidframe/rf_dagfuncs.c

DEFINITIONS

1. rf_ConfigureDAGFuncs
2. rf_TerminateFunc
3. rf_TerminateUndoFunc
4. rf_DiskReadMirrorIdleFunc
5. rf_DiskReadMirrorPartitionFunc
6. rf_DiskReadMirrorUndoFunc
7. rf_ParityLogUpdateFunc
8. rf_ParityLogOverwriteFunc
9. rf_ParityLogUpdateFunc
10. rf_ParityLogOverwriteFunc
11. rf_ParityLogUpdateUndoFunc
12. rf_ParityLogOverwriteUndoFunc
13. rf_NullNodeFunc
14. rf_NullNodeUndoFunc
15. rf_DiskReadFuncForThreads
16. rf_DiskWriteFuncForThreads
17. rf_DiskUndoFunc
18. rf_DiskUnlockFuncForThreads
19. rf_GenericWakeupFunc
20. rf_RegularXorFunc
21. rf_SimpleXorFunc
22. rf_RecoveryXorFunc
23. rf_XorIntoBuffer
24. rf_bxor
25. rf_longword_bxor
26. rf_longword_bxor3
27. rf_bxor3

    1 /*      $OpenBSD: rf_dagfuncs.c,v 1.7 2004/09/20 17:51:07 miod Exp $    */
    2 /*      $NetBSD: rf_dagfuncs.c,v 1.6 2000/03/30 12:45:40 augustss Exp $ */
    4 /*
    5  * Copyright (c) 1995 Carnegie-Mellon University.
    6  * All rights reserved.
    7  *
    8  * Author: Mark Holland, William V. Courtright II
    9  *
   10  * Permission to use, copy, modify and distribute this software and
   11  * its documentation is hereby granted, provided that both the copyright
   12  * notice and this permission notice appear in all copies of the
   13  * software, derivative works or modified versions, and any portions
   14  * thereof, and that both notices appear in supporting documentation.
   15  *
   16  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   17  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   18  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   19  *
   20  * Carnegie Mellon requests users of this software to return to
   21  *
   22  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   23  *  School of Computer Science
   24  *  Carnegie Mellon University
   25  *  Pittsburgh PA 15213-3890
   26  *
   27  * any improvements or extensions that they make and grant Carnegie the
   28  * rights to redistribute these changes.
   29  */
   31 /*
   32  * dagfuncs.c -- DAG node execution routines.
   33  *
   34  * Rules:
   35  * 1. Every DAG execution function must eventually cause node->status to
   36  *    get set to "good" or "bad", and "FinishNode" to be called. In the
   37  *    case of nodes that complete immediately (xor, NullNodeFunc, etc),
   38  *    the node execution function can do these two things directly. In
   39  *    the case of nodes that have to wait for some event (a disk read to
   40  *    complete, a lock to be released, etc) to occur before they can
   41  *    complete, this is typically achieved by having whatever module
   42  *    is doing the operation call GenericWakeupFunc upon completion.
   43  * 2. DAG execution functions should check the status in the DAG header
   44  *    and NOP out their operations if the status is not "enable". However,
   45  *    execution functions that release resources must be sure to release
   46  *    them even when they NOP out the function that would use them.
   47  *    Functions that acquire resources should go ahead and acquire them
   48  *    even when they NOP, so that a downstream release node will not have
   49  *    to check to find out whether or not the acquire was suppressed.
   50  */
   52 #include <sys/ioctl.h>
   53 #include <sys/param.h>
   55 #include "rf_archs.h"
   56 #include "rf_raid.h"
   57 #include "rf_dag.h"
   58 #include "rf_layout.h"
   59 #include "rf_etimer.h"
   60 #include "rf_acctrace.h"
   61 #include "rf_diskqueue.h"
   62 #include "rf_dagfuncs.h"
   63 #include "rf_general.h"
   64 #include "rf_engine.h"
   65 #include "rf_dagutils.h"
   67 #include "rf_kintf.h"
   69 #if     RF_INCLUDE_PARITYLOGGING > 0
   70 #include "rf_paritylog.h"
   71 #endif  /* RF_INCLUDE_PARITYLOGGING > 0 */
   73 int     (*rf_DiskReadFunc) (RF_DagNode_t *);
   74 int     (*rf_DiskWriteFunc) (RF_DagNode_t *);
   75 int     (*rf_DiskReadUndoFunc) (RF_DagNode_t *);
   76 int     (*rf_DiskWriteUndoFunc) (RF_DagNode_t *);
   77 int     (*rf_DiskUnlockFunc) (RF_DagNode_t *);
   78 int     (*rf_DiskUnlockUndoFunc) (RF_DagNode_t *);
   79 int     (*rf_RegularXorUndoFunc) (RF_DagNode_t *);
   80 int     (*rf_SimpleXorUndoFunc) (RF_DagNode_t *);
   81 int     (*rf_RecoveryXorUndoFunc) (RF_DagNode_t *);
   83 /*****************************************************************************
   84  * Main (only) configuration routine for this module.
   85  *****************************************************************************/
   86 int
rf_ConfigureDAGFuncs(RF_ShutdownList_t **listp)
   88 {
RF_ASSERT(((sizeof(long) == 8) && RF_LONGSHIFT == 3) ||
   90             ((sizeof(long) == 4) && RF_LONGSHIFT == 2));
rf_DiskReadFunc = rf_DiskReadFuncForThreads;
rf_DiskReadUndoFunc = rf_DiskUndoFunc;
rf_DiskWriteFunc = rf_DiskWriteFuncForThreads;
rf_DiskWriteUndoFunc = rf_DiskUndoFunc;
rf_DiskUnlockFunc = rf_DiskUnlockFuncForThreads;
rf_DiskUnlockUndoFunc = rf_NullNodeUndoFunc;
rf_RegularXorUndoFunc = rf_NullNodeUndoFunc;
rf_SimpleXorUndoFunc = rf_NullNodeUndoFunc;
rf_RecoveryXorUndoFunc = rf_NullNodeUndoFunc;
  100         return (0);
  101 }
  104 /*****************************************************************************
  105  * The execution function associated with a terminate node.
  106  *****************************************************************************/
  107 int
rf_TerminateFunc(RF_DagNode_t *node)
  109 {
RF_ASSERT(node->dagHdr->numCommits == node->dagHdr->numCommitNodes);
node->status = rf_good;
  112         return (rf_FinishNode(node, RF_THREAD_CONTEXT));
  113 }
  115 int
rf_TerminateUndoFunc(RF_DagNode_t *node)
  117 {
  118         return (0);
  119 }
  122 /*****************************************************************************
  123  * Execution functions associated with a mirror node.
  124  *
  125  * parameters:
  126  *
  127  * 0 - Physical disk address of data.
  128  * 1 - Buffer for holding read data.
  129  * 2 - Parity stripe ID.
  130  * 3 - Flags.
  131  * 4 - Physical disk address of mirror (parity).
  132  *
  133  *****************************************************************************/
  135 int
rf_DiskReadMirrorIdleFunc(RF_DagNode_t *node)
  137 {
  138         /*
  139          * Select the mirror copy with the shortest queue and fill in node
  140          * parameters with physical disk address.
  141          */
rf_SelectMirrorDiskIdle(node);
  144         return (rf_DiskReadFunc(node));
  145 }
  147 int
rf_DiskReadMirrorPartitionFunc(RF_DagNode_t *node)
  149 {
  150         /*
  151          * Select the mirror copy with the shortest queue and fill in node
  152          * parameters with physical disk address.
  153          */
rf_SelectMirrorDiskPartition(node);
  156         return (rf_DiskReadFunc(node));
  157 }
  159 int
rf_DiskReadMirrorUndoFunc(RF_DagNode_t *node)
  161 {
  162         return (0);
  163 }
  167 #if     RF_INCLUDE_PARITYLOGGING > 0
  168 /*****************************************************************************
  169  * The execution function associated with a parity log update node.
  170  *****************************************************************************/
  171 int
rf_ParityLogUpdateFunc(RF_DagNode_t *node)
  173 {
RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
caddr_t buf = (caddr_t) node->params[1].p;
RF_ParityLogData_t *logData;
RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
RF_Etimer_t timer;
  180         if (node->dagHdr->status == rf_enable) {
RF_ETIMER_START(timer);
logData = rf_CreateParityLogData(RF_UPDATE, pda, buf,
  183                     (RF_Raid_t *) (node->dagHdr->raidPtr),
node->wakeFunc, (void *) node,
node->dagHdr->tracerec, timer);
  186                 if (logData)
rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
  188                 else {
RF_ETIMER_STOP(timer);
RF_ETIMER_EVAL(timer);
tracerec->plog_us += RF_ETIMER_VAL_US(timer);
  192                         (node->wakeFunc) (node, ENOMEM);
  193                 }
  194         }
  195         return (0);
  196 }
  199 /*****************************************************************************
  200  * The execution function associated with a parity log overwrite node.
  201  *****************************************************************************/
  202 int
rf_ParityLogOverwriteFunc(RF_DagNode_t *node)
  204 {
RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
caddr_t buf = (caddr_t) node->params[1].p;
RF_ParityLogData_t *logData;
RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
RF_Etimer_t timer;
  211         if (node->dagHdr->status == rf_enable) {
RF_ETIMER_START(timer);
logData = rf_CreateParityLogData(RF_OVERWRITE, pda, buf,
  214                     (RF_Raid_t *) (node->dagHdr->raidPtr), node->wakeFunc,
  215                     (void *) node, node->dagHdr->tracerec, timer);
  216                 if (logData)
rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
  218                 else {
RF_ETIMER_STOP(timer);
RF_ETIMER_EVAL(timer);
tracerec->plog_us += RF_ETIMER_VAL_US(timer);
  222                         (node->wakeFunc) (node, ENOMEM);
  223                 }
  224         }
  225         return (0);
  226 }
  227 #else   /* RF_INCLUDE_PARITYLOGGING > 0 */
  229 int
rf_ParityLogUpdateFunc(RF_DagNode_t *node)
  231 {
  232         return (0);
  233 }
  235 int
rf_ParityLogOverwriteFunc(RF_DagNode_t *node)
  237 {
  238         return (0);
  239 }
  240 #endif  /* RF_INCLUDE_PARITYLOGGING > 0 */
  242 int
rf_ParityLogUpdateUndoFunc(RF_DagNode_t *node)
  244 {
  245         return (0);
  246 }
  248 int
rf_ParityLogOverwriteUndoFunc(RF_DagNode_t *node)
  250 {
  251         return (0);
  252 }
  254 /*****************************************************************************
  255  * The execution function associated with a NOP node.
  256  *****************************************************************************/
  257 int
rf_NullNodeFunc(RF_DagNode_t *node)
  259 {
node->status = rf_good;
  261         return (rf_FinishNode(node, RF_THREAD_CONTEXT));
  262 }
  264 int
rf_NullNodeUndoFunc(RF_DagNode_t *node)
  266 {
node->status = rf_undone;
  268         return (rf_FinishNode(node, RF_THREAD_CONTEXT));
  269 }
  272 /*****************************************************************************
  273  * The execution function associated with a disk-read node.
  274  *****************************************************************************/
  275 int
rf_DiskReadFuncForThreads(RF_DagNode_t *node)
  277 {
RF_DiskQueueData_t *req;
RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
caddr_t buf = (caddr_t) node->params[1].p;
RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
  282         unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
  283         unsigned lock = RF_EXTRACT_LOCK_FLAG(node->params[3].v);
  284         unsigned unlock = RF_EXTRACT_UNLOCK_FLAG(node->params[3].v);
  285         unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
RF_DiskQueueDataFlags_t flags = 0;
RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ?
RF_IO_TYPE_READ : RF_IO_TYPE_NOP;
RF_DiskQueue_t **dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
  290         void *b_proc = NULL;
  292         if (node->dagHdr->bp)
b_proc = (void *) ((struct buf *) node->dagHdr->bp)->b_proc;
RF_ASSERT(!(lock && unlock));
flags |= (lock) ? RF_LOCK_DISK_QUEUE : 0;
flags |= (unlock) ? RF_UNLOCK_DISK_QUEUE : 0;
req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
buf, parityStripeID, which_ru,
  301             (int (*) (void *, int)) node->wakeFunc,
node, NULL, node->dagHdr->tracerec,
  303             (void *) (node->dagHdr->raidPtr), flags, b_proc);
  304         if (!req) {
  305                 (node->wakeFunc) (node, ENOMEM);
  306         } else {
node->dagFuncData = (void *) req;
rf_DiskIOEnqueue(&(dqs[pda->row][pda->col]), req, priority);
  309         }
  310         return (0);
  311 }
  314 /*****************************************************************************
  315  * the execution function associated with a disk-write node
  316  *****************************************************************************/
  317 int
rf_DiskWriteFuncForThreads(RF_DagNode_t *node)
  319 {
RF_DiskQueueData_t *req;
RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
caddr_t buf = (caddr_t) node->params[1].p;
RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
  324         unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
  325         unsigned lock = RF_EXTRACT_LOCK_FLAG(node->params[3].v);
  326         unsigned unlock = RF_EXTRACT_UNLOCK_FLAG(node->params[3].v);
  327         unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
RF_DiskQueueDataFlags_t flags = 0;
RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ?
RF_IO_TYPE_WRITE : RF_IO_TYPE_NOP;
RF_DiskQueue_t **dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
  332         void *b_proc = NULL;
  334         if (node->dagHdr->bp)
b_proc = (void *) ((struct buf *) node->dagHdr->bp)->b_proc;
  337         /* Normal processing (rollaway or forward recovery) begins here. */
RF_ASSERT(!(lock && unlock));
flags |= (lock) ? RF_LOCK_DISK_QUEUE : 0;
flags |= (unlock) ? RF_UNLOCK_DISK_QUEUE : 0;
req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
buf, parityStripeID, which_ru,
  343             (int (*) (void *, int)) node->wakeFunc, (void *) node, NULL,
node->dagHdr->tracerec, (void *) (node->dagHdr->raidPtr),
flags, b_proc);
  347         if (!req) {
  348                 (node->wakeFunc) (node, ENOMEM);
  349         } else {
node->dagFuncData = (void *) req;
rf_DiskIOEnqueue(&(dqs[pda->row][pda->col]), req, priority);
  352         }
  354         return (0);
  355 }
  356 /*****************************************************************************
  357  * The undo function for disk nodes.
  358  * Note:  This is not a proper undo of a write node, only locks are released.
  359  *        old data is not restored to disk !
  360  *****************************************************************************/
  361 int
rf_DiskUndoFunc(RF_DagNode_t *node)
  363 {
RF_DiskQueueData_t *req;
RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
RF_DiskQueue_t **dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP, 0L, 0, NULL, 0L, 0,
  369             (int (*) (void *, int)) node->wakeFunc, (void *) node,
NULL, node->dagHdr->tracerec, (void *) (node->dagHdr->raidPtr),
RF_UNLOCK_DISK_QUEUE, NULL);
  372         if (!req)
  373                 (node->wakeFunc) (node, ENOMEM);
  374         else {
node->dagFuncData = (void *) req;
rf_DiskIOEnqueue(&(dqs[pda->row][pda->col]), req,
RF_IO_NORMAL_PRIORITY);
  378         }
  380         return (0);
  381 }
  383 /*****************************************************************************
  384  * The execution function associated with an "unlock disk queue" node.
  385  *****************************************************************************/
  386 int
rf_DiskUnlockFuncForThreads(RF_DagNode_t *node)
  388 {
RF_DiskQueueData_t *req;
RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
RF_DiskQueue_t **dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP, 0L, 0, NULL, 0L, 0,
  394             (int (*) (void *, int)) node->wakeFunc, (void *) node,
NULL, node->dagHdr->tracerec, (void *) (node->dagHdr->raidPtr),
RF_UNLOCK_DISK_QUEUE, NULL);
  397         if (!req)
  398                 (node->wakeFunc) (node, ENOMEM);
  399         else {
node->dagFuncData = (void *) req;
rf_DiskIOEnqueue(&(dqs[pda->row][pda->col]), req,
RF_IO_NORMAL_PRIORITY);
  403         }
  405         return (0);
  406 }
  408 /*****************************************************************************
  409  * Callback routine for DiskRead and DiskWrite nodes. When the disk op
  410  * completes, the routine is called to set the node status and inform
  411  * the execution engine that the node has fired.
  412  *****************************************************************************/
  413 int
rf_GenericWakeupFunc(RF_DagNode_t *node, int status)
  415 {
  416         switch (node->status) {
  417         case rf_bwd1:
node->status = rf_bwd2;
  419                 if (node->dagFuncData)
rf_FreeDiskQueueData((RF_DiskQueueData_t *)
node->dagFuncData);
  422                 return (rf_DiskWriteFuncForThreads(node));
  423                 break;
  424         case rf_fired:
  425                 if (status)
node->status = rf_bad;
  427                 else
node->status = rf_good;
  429                 break;
  430         case rf_recover:
  431                 /* Probably should never reach this case. */
  432                 if (status)
node->status = rf_panic;
  434                 else
node->status = rf_undone;
  436                 break;
  437         default:
printf("rf_GenericWakeupFunc:");
printf("node->status is %d,", node->status);
printf("status is %d \n", status);
RF_PANIC();
  442                 break;
  443         }
  444         if (node->dagFuncData)
rf_FreeDiskQueueData((RF_DiskQueueData_t *) node->dagFuncData);
  446         return (rf_FinishNode(node, RF_INTR_CONTEXT));
  447 }
  450 /*****************************************************************************
  451  * There are three distinct types of xor nodes.
  452  *
  453  * A "regular xor" is used in the fault-free case where the access spans
  454  * a complete stripe unit. It assumes that the result buffer is one full
  455  * stripe unit in size, and uses the stripe-unit-offset values that it
  456  * computes from the PDAs to determine where within the stripe unit to
  457  * XOR each argument buffer.
  458  *
  459  * A "simple xor" is used in the fault-free case where the access touches
  460  * only a portion of one (or two, in some cases) stripe unit(s). It assumes
  461  * that all the argument buffers are of the same size and have the same
  462  * stripe unit offset.
  463  *
  464  * A "recovery xor" is used in the degraded-mode case. It's similar to
  465  * the regular xor function except that it takes the failed PDA as an
  466  * additional parameter, and uses it to determine what portions of the
  467  * argument buffers need to be xor'd into the result buffer, and where
  468  * in the result buffer they should go.
  469  *****************************************************************************/
  471 /*
  472  * Xor the params together and store the result in the result field.
  473  * Assume the result field points to a buffer that is the size of one SU,
  474  * and use the pda params to determine where within the buffer to XOR
  475  * the input buffers.
  476  */
  477 int
rf_RegularXorFunc(RF_DagNode_t *node)
  479 {
RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
RF_Etimer_t timer;
  483         int i, retcode;
retcode = 0;
  486         if (node->dagHdr->status == rf_enable) {
  487                 /* Don't do the XOR if the input is the same as the output. */
RF_ETIMER_START(timer);
  489                 for (i = 0; i < node->numParams - 1; i += 2)
  490                         if (node->params[i + 1].p != node->results[0]) {
retcode = rf_XorIntoBuffer(raidPtr,
  492                                     (RF_PhysDiskAddr_t *) node->params[i].p,
  493                                     (char *) node->params[i + 1].p,
  494                                     (char *) node->results[0],
node->dagHdr->bp);
  496                         }
RF_ETIMER_STOP(timer);
RF_ETIMER_EVAL(timer);
tracerec->xor_us += RF_ETIMER_VAL_US(timer);
  500         }
  501         /* Call wake func explicitly since no I/O in this node. */
  502         return (rf_GenericWakeupFunc(node, retcode));
  503 }
  505 /* Xor the inputs into the result buffer, ignoring placement issues. */
  506 int
rf_SimpleXorFunc(RF_DagNode_t *node)
  508 {
RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
  510         int i, retcode = 0;
RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
RF_Etimer_t timer;
  514         if (node->dagHdr->status == rf_enable) {
RF_ETIMER_START(timer);
  516                 /* Don't do the XOR if the input is the same as the output. */
  517                 for (i = 0; i < node->numParams - 1; i += 2)
  518                         if (node->params[i + 1].p != node->results[0]) {
retcode = rf_bxor((char *)
node->params[i + 1].p,
  521                                     (char *) node->results[0],
rf_RaidAddressToByte(raidPtr,
  523                                     ((RF_PhysDiskAddr_t *)
node->params[i].p)->numSector),
  525                                     (struct buf *) node->dagHdr->bp);
  526                         }
RF_ETIMER_STOP(timer);
RF_ETIMER_EVAL(timer);
tracerec->xor_us += RF_ETIMER_VAL_US(timer);
  530         }
  531         /* Call wake func explicitly since no I/O in this node. */
  532         return (rf_GenericWakeupFunc(node, retcode));
  533 }
  535 /*
  536  * This xor is used by the degraded-mode dag functions to recover lost data.
  537  * The second-to-last parameter is the PDA for the failed portion of the access.
  538  * The code here looks at this PDA and assumes that the xor target buffer is
  539  * equal in size to the number of sectors in the failed PDA. It then uses
  540  * the other PDAs in the parameter list to determine where within the target
  541  * buffer the corresponding data should be xored.
  542  */
  543 int
rf_RecoveryXorFunc(RF_DagNode_t *node)
  545 {
RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) & raidPtr->Layout;
RF_PhysDiskAddr_t *failedPDA =
  549             (RF_PhysDiskAddr_t *) node->params[node->numParams - 2].p;
  550         int i, retcode = 0;
RF_PhysDiskAddr_t *pda;
  552         int suoffset, failedSUOffset =
rf_StripeUnitOffset(layoutPtr, failedPDA->startSector);
  554         char *srcbuf, *destbuf;
RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
RF_Etimer_t timer;
  558         if (node->dagHdr->status == rf_enable) {
RF_ETIMER_START(timer);
  560                 for (i = 0; i < node->numParams - 2; i += 2)
  561                         if (node->params[i + 1].p != node->results[0]) {
pda = (RF_PhysDiskAddr_t *) node->params[i].p;
srcbuf = (char *) node->params[i + 1].p;
suoffset = rf_StripeUnitOffset(layoutPtr,
pda->startSector);
destbuf = ((char *) node->results[0]) +
rf_RaidAddressToByte(raidPtr,
suoffset - failedSUOffset);
retcode = rf_bxor(srcbuf, destbuf,
rf_RaidAddressToByte(raidPtr,
pda->numSector), node->dagHdr->bp);
  572                         }
RF_ETIMER_STOP(timer);
RF_ETIMER_EVAL(timer);
tracerec->xor_us += RF_ETIMER_VAL_US(timer);
  576         }
  577         return (rf_GenericWakeupFunc(node, retcode));
  578 }
  581 /*****************************************************************************
  582  * The next three functions are utilities used by the above xor-execution
  583  * functions.
  584  *****************************************************************************/
  586 /*
  587  * This is just a glorified buffer xor. Targbuf points to a buffer that is
  588  * one full stripe unit in size. srcbuf points to a buffer that may be less
  589  * than 1 SU, but never more. When the access described by pda is one SU in
  590  * size (which by implication means it's SU-aligned), all that happens is
  591  * (targbuf) <- (srcbuf ^ targbuf). When the access is less than one SU in
  592  * size the XOR occurs on only the portion of targbuf identified in the pda.
  593  */
  595 int
rf_XorIntoBuffer(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda, char *srcbuf,
  597     char *targbuf, void *bp)
  598 {
  599         char *targptr;
  600         int sectPerSU = raidPtr->Layout.sectorsPerStripeUnit;
  601         int SUOffset = pda->startSector % sectPerSU;
  602         int length, retcode = 0;
RF_ASSERT(pda->numSector <= sectPerSU);
targptr = targbuf + rf_RaidAddressToByte(raidPtr, SUOffset);
length = rf_RaidAddressToByte(raidPtr, pda->numSector);
retcode = rf_bxor(srcbuf, targptr, length, bp);
  609         return (retcode);
  610 }
  612 /*
  613  * It really should be the case that the buffer pointers (returned by malloc)
  614  * are aligned to the natural word size of the machine, so this is the only
  615  * case we optimize for. The length should always be a multiple of the sector
  616  * size, so there should be no problem with leftover bytes at the end.
  617  */
  618 int
rf_bxor(char *src, char *dest, int len, void *bp)
  620 {
  621         unsigned mask = sizeof(long) - 1, retcode = 0;
  623         if (!(((unsigned long) src) & mask) &&
  624             !(((unsigned long) dest) & mask) && !(len & mask)) {
retcode = rf_longword_bxor((unsigned long *) src,
  626                     (unsigned long *) dest, len >> RF_LONGSHIFT, bp);
  627         } else {
RF_ASSERT(0);
  629         }
  630         return (retcode);
  631 }
  633 /* Map a user buffer into kernel space, if necessary. */
  634 #define REMAP_VA(_bp,x,y)       (y) = (x)
  636 /*
  637  * When XORing in kernel mode, we need to map each user page to kernel
  638  * space before we can access it.
  639  * We don't want to assume anything about which input buffers are in
  640  * kernel/user space, nor about their alignment, so in each loop we
  641  * compute the maximum number of bytes that we can xor without crossing
  642  * any page boundaries, and do only this many bytes before the next remap.
  643  */
  644 int
rf_longword_bxor(unsigned long *src, unsigned long *dest, int len, void *bp)
  646 {
  647         unsigned long *end = src + len; /* len in longwords. */
  648         unsigned long d0, d1, d2, d3, s0, s1, s2, s3; /* temps */
  649         unsigned long *pg_src, *pg_dest; /* Per-page source/dest pointers. */
  650         int longs_this_time; /* # longwords to xor in the current iteration. */
REMAP_VA(bp, src, pg_src);
REMAP_VA(bp, dest, pg_dest);
  654         if (!pg_src || !pg_dest)
  655                 return (EFAULT);
  657         while (len >= 4) {
longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(pg_src),
RF_BLIP(pg_dest)) >> RF_LONGSHIFT);
src += longs_this_time;
dest += longs_this_time;
len -= longs_this_time;
  663                 while (longs_this_time >= 4) {
d0 = pg_dest[0];
d1 = pg_dest[1];
d2 = pg_dest[2];
d3 = pg_dest[3];
s0 = pg_src[0];
s1 = pg_src[1];
s2 = pg_src[2];
s3 = pg_src[3];
pg_dest[0] = d0 ^ s0;
pg_dest[1] = d1 ^ s1;
pg_dest[2] = d2 ^ s2;
pg_dest[3] = d3 ^ s3;
pg_src += 4;
pg_dest += 4;
longs_this_time -= 4;
  679                 }
  680                 while (longs_this_time > 0) {
  681                         /* Cannot cross any page boundaries here. */
  682                         *pg_dest++ ^= *pg_src++;
longs_this_time--;
  684                 }
  686                 /*
  687                  * Either we're done, or we've reached a page boundary on one
  688                  * (or possibly both) of the pointers.
  689                  */
  690                 if (len) {
  691                         if (RF_PAGE_ALIGNED(src))
REMAP_VA(bp, src, pg_src);
  693                         if (RF_PAGE_ALIGNED(dest))
REMAP_VA(bp, dest, pg_dest);
  695                         if (!pg_src || !pg_dest)
  696                                 return (EFAULT);
  697                 }
  698         }
  699         while (src < end) {
  700                 *pg_dest++ ^= *pg_src++;
src++;
dest++;
len--;
  704                 if (RF_PAGE_ALIGNED(src))
REMAP_VA(bp, src, pg_src);
  706                 if (RF_PAGE_ALIGNED(dest))
REMAP_VA(bp, dest, pg_dest);
  708         }
RF_ASSERT(len == 0);
  710         return (0);
  711 }
  714 /*
  715  * dst = a ^ b ^ c;
  716  * a may equal dst
  717  * see comment above longword_bxor
  718  */
  719 int
rf_longword_bxor3(unsigned long *dst, unsigned long *a, unsigned long *b,
  721     unsigned long *c, int len, void *bp)
  722 {
  723         unsigned long a0, a1, a2, a3, b0, b1, b2, b3;
  724         /* Per-page source/dest pointers. */
  725         unsigned long *pg_a, *pg_b, *pg_c, *pg_dst;
  726         int longs_this_time;    /* # longs to xor in the current iteration */
  727         char dst_is_a = 0;
  729         /* Note: The length (len) is in longwords. */
REMAP_VA(bp, a, pg_a);
REMAP_VA(bp, b, pg_b);
REMAP_VA(bp, c, pg_c);
  734         if (a == dst) {
pg_dst = pg_a;
dst_is_a = 1;
  737         } else {
REMAP_VA(bp, dst, pg_dst);
  739         }
  741         /* Align dest to cache line. Can't cross a pg boundary on dst here. */
  742         while ((((unsigned long) pg_dst) & 0x1f)) {
  743                 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
dst++;
a++;
b++;
c++;
  748                 if (RF_PAGE_ALIGNED(a)) {
REMAP_VA(bp, a, pg_a);
  750                         if (!pg_a)
  751                                 return (EFAULT);
  752                 }
  753                 if (RF_PAGE_ALIGNED(b)) {
REMAP_VA(bp, a, pg_b);
  755                         if (!pg_b)
  756                                 return (EFAULT);
  757                 }
  758                 if (RF_PAGE_ALIGNED(c)) {
REMAP_VA(bp, a, pg_c);
  760                         if (!pg_c)
  761                                 return (EFAULT);
  762                 }
len--;
  764         }
  766         while (len > 4) {
longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(a),
RF_MIN(RF_BLIP(b), RF_MIN(RF_BLIP(c), RF_BLIP(dst)))) >>
RF_LONGSHIFT);
a += longs_this_time;
b += longs_this_time;
c += longs_this_time;
dst += longs_this_time;
len -= longs_this_time;
  775                 while (longs_this_time >= 4) {
a0 = pg_a[0];
longs_this_time -= 4;
a1 = pg_a[1];
a2 = pg_a[2];
a3 = pg_a[3];
pg_a += 4;
b0 = pg_b[0];
b1 = pg_b[1];
b2 = pg_b[2];
b3 = pg_b[3];
  790                         /* Start dual issue. */
a0 ^= b0;
b0 = pg_c[0];
pg_b += 4;
a1 ^= b1;
a2 ^= b2;
a3 ^= b3;
b1 = pg_c[1];
a0 ^= b0;
b2 = pg_c[2];
a1 ^= b1;
b3 = pg_c[3];
a2 ^= b2;
pg_dst[0] = a0;
a3 ^= b3;
pg_dst[1] = a1;
pg_c += 4;
pg_dst[2] = a2;
pg_dst[3] = a3;
pg_dst += 4;
  816                 }
  817                 while (longs_this_time > 0) {
  818                         /* Cannot cross any page boundaries here. */
  819                         *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
longs_this_time--;
  821                 }
  823                 if (len) {
  824                         if (RF_PAGE_ALIGNED(a)) {
REMAP_VA(bp, a, pg_a);
  826                                 if (!pg_a)
  827                                         return (EFAULT);
  828                                 if (dst_is_a)
pg_dst = pg_a;
  830                         }
  831                         if (RF_PAGE_ALIGNED(b)) {
REMAP_VA(bp, b, pg_b);
  833                                 if (!pg_b)
  834                                         return (EFAULT);
  835                         }
  836                         if (RF_PAGE_ALIGNED(c)) {
REMAP_VA(bp, c, pg_c);
  838                                 if (!pg_c)
  839                                         return (EFAULT);
  840                         }
  841                         if (!dst_is_a)
  842                                 if (RF_PAGE_ALIGNED(dst)) {
REMAP_VA(bp, dst, pg_dst);
  844                                         if (!pg_dst)
  845                                                 return (EFAULT);
  846                                 }
  847                 }
  848         }
  849         while (len) {
  850                 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
dst++;
a++;
b++;
c++;
  855                 if (RF_PAGE_ALIGNED(a)) {
REMAP_VA(bp, a, pg_a);
  857                         if (!pg_a)
  858                                 return (EFAULT);
  859                         if (dst_is_a)
pg_dst = pg_a;
  861                 }
  862                 if (RF_PAGE_ALIGNED(b)) {
REMAP_VA(bp, b, pg_b);
  864                         if (!pg_b)
  865                                 return (EFAULT);
  866                 }
  867                 if (RF_PAGE_ALIGNED(c)) {
REMAP_VA(bp, c, pg_c);
  869                         if (!pg_c)
  870                                 return (EFAULT);
  871                 }
  872                 if (!dst_is_a)
  873                         if (RF_PAGE_ALIGNED(dst)) {
REMAP_VA(bp, dst, pg_dst);
  875                                 if (!pg_dst)
  876                                         return (EFAULT);
  877                         }
len--;
  879         }
  880         return (0);
  881 }
  883 int
rf_bxor3(unsigned char *dst, unsigned char *a, unsigned char *b,
  885     unsigned char *c, unsigned long len, void *bp)
  886 {
RF_ASSERT(((RF_UL(dst) | RF_UL(a) | RF_UL(b) | RF_UL(c) | len) & 0x7)
  888             == 0);
  890         return (rf_longword_bxor3((unsigned long *) dst, (unsigned long *) a,
  891                 (unsigned long *) b, (unsigned long *) c, len >> RF_LONGSHIFT,
bp));
  893 }