/*++ Copyright (C) Microsoft Corporation, 2009 Module Name: io.c Abstract: This file contains the core logic for programming and completing IO commands into an AHCI controller Notes: Revision History: --*/ #if _MSC_VER >= 1200 #pragma warning(push) #endif #pragma warning(disable:4214) // bit field types other than int #pragma warning(disable:4201) // nameless struct/union #include "generic.h" ULONG GetSlotToActivate( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ ULONG TargetSlots ) /*++ Select Slot to activate based on maximum slot number allowed Called by: Activate Queue It performs: Affected Variables/Registers: none Return Value: --*/ { UCHAR activeCount = 0; UCHAR emptyCount; UCHAR requestCount; UCHAR lastActiveSlot; ULONG slotToActivate = 0; UCHAR i; // 1. Device's queue depth is smaller than Controller's NT_ASSERT(ChannelExtension->DeviceExtension[0].DeviceParameters.MaxDeviceQueueDepth <= ChannelExtension->AdapterExtension->CAP.NCS); // Count the number of slots already in use if (ChannelExtension->SlotManager.CommandsIssued > 0) { activeCount = NumberOfSetBits(ChannelExtension->SlotManager.CommandsIssued); } // 1.1 Check if all slots are active. if (activeCount >= ChannelExtension->DeviceExtension[0].DeviceParameters.MaxDeviceQueueDepth) { // If all possible slots are full, no matter what, return no work (0) return 0; } // 2. Look for any entry from last active slot requestCount = NumberOfSetBits(TargetSlots); lastActiveSlot = ChannelExtension->LastActiveSlot; emptyCount = ChannelExtension->DeviceExtension[0].DeviceParameters.MaxDeviceQueueDepth - activeCount; // 3.1 Look for any entry from last active slot for (i = lastActiveSlot; i <= ChannelExtension->AdapterExtension->CAP.NCS; i++) { if ((TargetSlots & (1 << i)) > 0) { slotToActivate |= (1 << i); emptyCount--; requestCount--; if (emptyCount == 0 || requestCount == 0) { ChannelExtension->LastActiveSlot = i; return slotToActivate; } } } // 3.2 Look for any entry from beginning to last active slot // Slot 0 is reserved for internal command for (i = 1 ; i <= lastActiveSlot; i++) { if ((TargetSlots & (1 << i)) > 0) { slotToActivate |= (1 << i); emptyCount--; requestCount--; if (emptyCount == 0 || requestCount == 0) { ChannelExtension->LastActiveSlot = i; return slotToActivate; } } } NT_ASSERT(FALSE); return slotToActivate; } UCHAR GetSingleIo( PAHCI_CHANNEL_EXTENSION ChannelExtension ) /*++ Selects an IO circularly from the programmed Single IO slice starting with the slot after the most recently programmed slot This ensures the issuing of the SingleIO is FIFO It assumes: Single IO are never high priority Called by: Activate Queue It performs: (overview) 1 Initialize 2 Chose the slot (details) 1.1 Initialize variables 2.1 Chose the slot circularly starting with CCS Affected Variables/Registers: none Return Value: Slot number of the oldest programmed Single IO Slice slot If no slots are programmed the tag returned is 0xFF --*/ { UCHAR limit; UCHAR i; PAHCI_ADAPTER_EXTENSION adapterExtension = ChannelExtension->AdapterExtension; // 1.1 Initialize variables limit = ChannelExtension->CurrentCommandSlot; // if there is internal request pending, always get it first. if ((ChannelExtension->SlotManager.SingleIoSlice & 1) > 0) { return 0; } // 2.1 Chose the slot circularly starting with CCS for (i = limit; i <= adapterExtension->CAP.NCS; i++) { if ((ChannelExtension->SlotManager.SingleIoSlice & (1 << i)) > 0) { return i; } } for (i = 0; i < limit; i++) { if ((ChannelExtension->SlotManager.SingleIoSlice & (1 << i)) > 0) { return i; } } return 0xff; } VOID AddQueue( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _Inout_ PSTORAHCI_QUEUE Queue, _In_ PSTORAGE_REQUEST_BLOCK Srb, _In_ ULONG Signature, _In_ UCHAR Tag ) /* Input Parameters: ChannelExtension Queue Srb Signature: used to mark the location of last Srb in history log Tag: bit 31 ~ 24, Queue->CurrentDepth: bit 23 ~ 0 */ { PVOID tempTail, foundSrb; ULONG srbsFound; UNREFERENCED_PARAMETER(ChannelExtension); StorPortQuerySystemTime(&(Queue->LastTimeStampAddQueue)); if (Queue->Tail == NULL) { Queue->Tail = (PVOID) Srb; Queue->Head = (PVOID) Srb; } else { tempTail = Queue->Tail; if (SrbGetNextSrb(tempTail) == NULL) { //Verify SRBs are not about to be severed from the Queue SrbSetNextSrb(tempTail, (PVOID)Srb); } Queue->Tail = (PVOID)Srb; } Queue->CurrentDepth++; if (Queue->Head == NULL) { //It is impossible for Head to be NULL here srbsFound = 0; } else { foundSrb = Queue->Head; srbsFound = 1; foundSrb = SrbGetNextSrb(foundSrb); while (foundSrb) { srbsFound++; foundSrb = SrbGetNextSrb(foundSrb); } } Queue->DepthHistory[Queue->DepthHistoryIndex] = ( (Tag << 24) | Queue->CurrentDepth ); Queue->DepthHistoryIndex++; Queue->DepthHistoryIndex %= 100; Queue->DepthHistory[Queue->DepthHistoryIndex] = Signature; if (Queue->CurrentDepth > Queue->DeepestDepth) { Queue->DeepestDepth = Queue->CurrentDepth; } return; } PSTORAGE_REQUEST_BLOCK RemoveQueue( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _Inout_ PSTORAHCI_QUEUE Queue, _In_ ULONG Signature, _In_ UCHAR Tag ) { PVOID nextSrb, foundSrb; ULONG srbsFound; UNREFERENCED_PARAMETER(ChannelExtension); // Check to see if the queue is empty if (Queue->Head == NULL) { return NULL; } StorPortQuerySystemTime(&(Queue->LastTimeStampRemoveQueue)); //if it is not empty, pop nextSrb = Queue->Head; Queue->Head = SrbGetNextSrb(nextSrb); SrbSetNextSrb(nextSrb, NULL); //This should have been NULL to start with, return it to it's NULL state if (Queue->Head == NULL) { Queue->Tail = NULL; } Queue->CurrentDepth--; if (Queue->Head == NULL) { srbsFound = 0; } else { foundSrb = Queue->Head; srbsFound = 1; foundSrb = SrbGetNextSrb(foundSrb); while (foundSrb) { srbsFound++; foundSrb = SrbGetNextSrb(foundSrb); } } Queue->DepthHistory[Queue->DepthHistoryIndex] = ( (Tag << 24) | Queue->CurrentDepth ); Queue->DepthHistoryIndex++; Queue->DepthHistoryIndex %= 100; Queue->DepthHistory[Queue->DepthHistoryIndex] = Signature; if (Queue->CurrentDepth > Queue->DeepestDepth) { Queue->DeepestDepth = Queue->CurrentDepth; } return (PSTORAGE_REQUEST_BLOCK)nextSrb; } BOOLEAN ActivateQueue( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ BOOLEAN AtDIRQL ) /*++ It assumes: Called by: It performs: (overview) 1. Ensures the channel is ready for programming 2. Program the next IO according to the priority scheme (details) 1.1 Initialize variables 1.2 If the programming should not happen now, leave, ActivateQueue will be called again when these conditions are changed 2.1 Choose the Queue with which to program the controller Algorithm: 2.1.1 Single IO SRBs (including Request Sense and non data control commands) have highest priority. 2.1.2 When there are no Single IO commands, Normal IO get the next highest priority 2.1.3 When there are no Single or Normal commands, NCQ commands get the next highest priority 2.1.4 In the case that no IO is present in any Slices, program nothing 2.2 Program all the IO from the chosen queue into the controller Affected Variables/Registers: channelExtension CI SACT Return Values: --*/ { BOOLEAN status = TRUE; ULONG slotsToActivate = 0; BOOLEAN activateNcq = FALSE; int i; PAHCI_ADAPTER_EXTENSION adapterExtension = ChannelExtension->AdapterExtension; STOR_LOCK_HANDLE lockhandle = { InterruptLock, 0 }; // 1.1 Initialize variables if (LogExecuteFullDetail(adapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x00000022);//ActivateQueue } // If the programming should not happen now, leave if (ChannelExtension->StartState.ChannelNextStartState != StartComplete) { RecordExecutionHistory(ChannelExtension, 0x10010022);//ActivateQueue, Channel Not Yet Started status = FALSE; return status; } if (ChannelExtension->StateFlags.QueuePaused == TRUE) { RecordExecutionHistory(ChannelExtension, 0x10020022);//ActivateQueue, Channel Queue Paused status = FALSE; return status; } if (!IsPortStartCapable(ChannelExtension)) { RecordExecutionHistory(ChannelExtension, 0x10030022);//ActivateQueue, Channel Not Start Capable status = FALSE; return status; } if (ErrorRecoveryIsPending(ChannelExtension)) { RecordExecutionHistory(ChannelExtension, 0x10070022);//ActivateQueue, Error Recovery is pending. status = FALSE; return status; } if (!AtDIRQL) { AhciInterruptSpinlockAcquire(ChannelExtension->AdapterExtension, ChannelExtension->PortNumber, &lockhandle); } // 1.1.1 If there is no command to program, leave if ((ChannelExtension->SlotManager.SingleIoSlice == 0) && (ChannelExtension->SlotManager.NormalQueueSlice == 0) && (ChannelExtension->SlotManager.NCQueueSlice == 0)) { if (LogExecuteFullDetail(adapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x10040022);//ActivateQueue, No Commands to program } status = FALSE; goto Exit; } if ((ChannelExtension->StateFlags.ReservedSlotInUse == 1) && ((ChannelExtension->SlotManager.SingleIoSlice & 1) == 0)) { //In case of device initialization is in process but the local command is not ready yet, do not program other commands. if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x10080022);//ActivateQueue, an init command is in process. } status = FALSE; goto Exit; } // 2.1 Choose the Queue with which to program the controller // 2.1.1 Single IO SRBs have highest priority. if (ChannelExtension->SlotManager.SingleIoSlice != 0) { if ((ChannelExtension->SlotManager.NCQueueSliceIssued | ChannelExtension->SlotManager.NormalQueueSliceIssued | ChannelExtension->SlotManager.SingleIoSliceIssued) == 0) { // Safely get Single IO in round robin fashion i = GetSingleIo(ChannelExtension); if (i != 0xff) { PSLOT_CONTENT slotContent = &ChannelExtension->Slot[i]; slotsToActivate = (1 << i); ChannelExtension->SlotManager.SingleIoSlice &= ~slotsToActivate; ChannelExtension->SlotManager.SingleIoSliceIssued |= slotsToActivate; ChannelExtension->StateFlags.QueuePaused = TRUE; //and pause the queue so no other IO get programmed if (LogExecuteFullDetail(adapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x10090022); //ActivateQueue, Single IO pause queue } // 2.1.1.1 Update the Identify command for enumeration. This needs to be here for PxSIG to be accessible. // Accessing PxSIG register here to avoid invalid values from time when the port is being reset or not started, or other scenarios. if (slotContent->Srb != NULL) { PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(slotContent->Srb); PAHCI_COMMAND_TABLE cmdTable = (PAHCI_COMMAND_TABLE)srbExtension; if ((srbExtension->AtaFunction == ATA_FUNCTION_ATA_IDENTIFY) && (srbExtension->CompletionRoutine == AhciPortIdentifyDevice) && (srbExtension->TaskFile.Current.bCommandReg == IDE_COMMAND_NOT_VALID) && (cmdTable->CFIS.Command == IDE_COMMAND_NOT_VALID) ) { // For enumeration command, it's safe now to check the PxSIG register as starting port has been executed. ULONG sig = 0; sig = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SIG.AsUlong); if (sig == ATA_DEVICE_SIGNATURE_ATA) { //ATA srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_IDENTIFY; cmdTable->CFIS.Command = IDE_COMMAND_IDENTIFY; } else { //ATAPI srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_ATAPI_IDENTIFY; cmdTable->CFIS.Command = IDE_COMMAND_ATAPI_IDENTIFY; } } } else { NT_ASSERT(FALSE); } } } // 2.1.2 When there are no Single IO commands, Normal IO get the next highest priority } else if (ChannelExtension->SlotManager.NormalQueueSlice != 0) { // Normal commands can not be sent when NCQ commands are outstanding. When the NCQ commands complete ActivateQueue will get called again. if (ChannelExtension->SlotManager.NCQueueSliceIssued == 0) { // Grab the High Priority Normal IO before the Low Priority Normal IO slotsToActivate = ChannelExtension->SlotManager.HighPriorityAttribute & ChannelExtension->SlotManager.NormalQueueSlice; // If there aren't any High Priority, grab everything else if (slotsToActivate > 0) { ChannelExtension->SlotManager.NormalQueueSlice &= ~slotsToActivate; } else { slotsToActivate = ChannelExtension->SlotManager.NormalQueueSlice; ChannelExtension->SlotManager.NormalQueueSlice = 0; } ChannelExtension->SlotManager.NormalQueueSliceIssued |= slotsToActivate; } // 2.1.3 When there are no Single or Normal commands, NCQ commands get the next highest priority } else if ((ChannelExtension->SlotManager.NCQueueSlice != 0) && (ChannelExtension->StateFlags.NcqErrorRecoveryInProcess == 0)) { // NCQ commands can not be sent when Normal commands are outstanding. When the Normal commands complete, Activate Queue will get called again. if ((ChannelExtension->SlotManager.SingleIoSliceIssued | ChannelExtension->SlotManager.NormalQueueSliceIssued ) != 0) { slotsToActivate = 0; } else { // Grab the High Priority NCQ IO before the Low Priority NCQ IO slotsToActivate = ChannelExtension->SlotManager.HighPriorityAttribute & ChannelExtension->SlotManager.NCQueueSlice; // If there aren't any High Priority, grab everything else if (slotsToActivate == 0) { slotsToActivate = ChannelExtension->SlotManager.NCQueueSlice; } // And apply any device outstanding IO limits to filter down which IO to activate if (slotsToActivate > 0) { if (ChannelExtension->DeviceExtension[0].DeviceParameters.MaxDeviceQueueDepth < ChannelExtension->MaxPortQueueDepth) { // Get allowed slots if the device queue depth is less than the port can support. slotsToActivate = GetSlotToActivate(ChannelExtension, slotsToActivate); } if (slotsToActivate > 0) { // And if there are any IO still selected, clear them from the NCQueue activateNcq = TRUE; //Remember to program SACT for these commands ChannelExtension->SlotManager.NCQueueSlice &= ~slotsToActivate; ChannelExtension->SlotManager.NCQueueSliceIssued |= slotsToActivate; // The selected IO will be activated at the end of this function } } } // 2.1.4 In the case that no IO is present in any Slices, program nothing } // 2.1 Program all the IO from the chosen queue into the controller if (slotsToActivate != 0) { // 2.2 Get command start time if (adapterExtension->TracingEnabled) { LARGE_INTEGER perfCounter = {0}; ULONG pendingProgrammingCommands = slotsToActivate; i = 0; StorPortQueryPerformanceCounter((PVOID)adapterExtension, NULL, &perfCounter); while (pendingProgrammingCommands) { // TODO: there is potential race condition that causes slot flags mismatch and Srb is NULL here. if ((pendingProgrammingCommands & 1) && (ChannelExtension->Slot[i].Srb != NULL)) { PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(ChannelExtension->Slot[i].Srb); srbExtension->StartTime = perfCounter.QuadPart; } i++; pendingProgrammingCommands >>= 1; } } ChannelExtension->SlotManager.CommandsIssued |= slotsToActivate; // Program registers if (activateNcq) { StorPortWriteRegisterUlong(adapterExtension, &ChannelExtension->Px->SACT, slotsToActivate); } StorPortWriteRegisterUlong(adapterExtension, &ChannelExtension->Px->CI, slotsToActivate); } if (LogExecuteFullDetail(adapterExtension->LogFlags)) { RecordInterruptHistory(ChannelExtension, slotsToActivate, 0, 0, (ChannelExtension->SlotManager.SingleIoSliceIssued | ChannelExtension->SlotManager.NormalQueueSliceIssued), ChannelExtension->SlotManager.NCQueueSliceIssued, 0x10060022); //ActivateQueue, after getting IO slices, RecordExecutionHistory(ChannelExtension, 0x10050022);//Exit ActivateQueue } status = TRUE; Exit: if (!AtDIRQL) { AhciInterruptSpinlockRelease(ChannelExtension->AdapterExtension, ChannelExtension->PortNumber, &lockhandle); } return status; } __inline ULONGLONG CalculateTimeDurationIn100ns ( _In_ ULONGLONG TimeDuration, _In_ ULONGLONG CounterFrequency ) { ULONGLONG timeIn100ns = 0; // As the counter can hold more than 20,000 years for the same power cycle, // the situation about the counter start over again is considered an error case of inputs. if (CounterFrequency > 0) { // Difference between performance counters, needs to convert to 100ns. ULONGLONG countersDiff = TimeDuration; // Get seconds timeIn100ns = countersDiff / CounterFrequency; // Get milliseconds countersDiff = (countersDiff % CounterFrequency) * 1000; timeIn100ns *= 1000; timeIn100ns += countersDiff / CounterFrequency; // Get 100 nanoseconds countersDiff = (countersDiff % CounterFrequency) * 10000; timeIn100ns *= 10000; timeIn100ns += countersDiff / CounterFrequency; } return timeIn100ns; } VOID AhciCompleteIssuedSRBs( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ UCHAR SrbStatus, _In_ BOOLEAN AtDIRQL ) /*++ This function completes a command that has been programmed into a Slot It assumes: The command in not running on the AHCI controller ATAPI commands will be sent 1 at a time Called by: AhciHwInterrupt AhciPortReset AhciNonQueuedErrorRecovery It performs: (overview) 1 Initialize 2 Process all commands in CommandsToComplete 3 Start the next batch of commands (details) 1.1 Initialize variables 2.1 For every command marked as completed 2.2 Set the status 2.3 Monitor to see that any NCQ commands are completing 2.4 Give the slot back 3.1 Start the next IO(s) if any Affected Variables/Registers: ChannelExtension->Slot[srbExtension->QueueTag].Srb ChannelExtension->SlotManager.CommandsToComplete --*/ { PSLOT_CONTENT slotContent; UCHAR i; PAHCI_ADAPTER_EXTENSION adapterExtension; PAHCI_SRB_EXTENSION srbExtension; LARGE_INTEGER perfCounter = { 0 }; LARGE_INTEGER perfFrequency = { 0 }; // 1.1 Initialize variables adapterExtension = ChannelExtension->AdapterExtension; if (LogExecuteFullDetail(adapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x00000046);//AhciCompleteIssuedSRBs } if (adapterExtension->TracingEnabled && (ChannelExtension->SlotManager.CommandsToComplete)) { StorPortQueryPerformanceCounter((PVOID)adapterExtension, &perfFrequency, &perfCounter); } // 2.1 For every command marked as completed for (i = 0; i <= (adapterExtension->CAP.NCS); i++) { if ((ChannelExtension->SlotManager.CommandsToComplete & (1 << i)) > 0) { slotContent = &ChannelExtension->Slot[i]; if (slotContent->Srb == NULL) { // This shall never happen. // The completed slot has no SRB so it can not be completed back to Storport. // Give back the empty slot NT_ASSERT(FALSE); ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << i); ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << i); continue; } if (slotContent->CmdHeader == NULL) { // This shall never happen. // Give back the empty slot NT_ASSERT(FALSE); ChannelExtension->SlotManager.CommandsToComplete &= ~(1 << i); ChannelExtension->SlotManager.HighPriorityAttribute &= ~(1 << i); // It is now impossible to determine if a data transfer completed correctly slotContent->Srb->SrbStatus = SRB_STATUS_ABORTED; AhciCompleteRequest(ChannelExtension, slotContent->Srb, AtDIRQL); continue; } srbExtension = GetSrbExtension(slotContent->Srb); // 2.1.2 Log command execution time, if it's allowed if (adapterExtension->TracingEnabled && (srbExtension->StartTime != 0) && (perfCounter.QuadPart != 0) && !IsMiniportInternalSrb(ChannelExtension, slotContent->Srb)) { ULONGLONG durationTime = CalculateTimeDurationIn100ns((perfCounter.QuadPart - srbExtension->StartTime), perfFrequency.QuadPart); StorPortNotification(IoTargetRequestServiceTime, (PVOID)adapterExtension, durationTime, slotContent->Srb); } // 2.2 Set the status if ((SrbStatus == SRB_STATUS_SUCCESS) && (!IsRequestSenseSrb(srbExtension->AtaFunction)) && (srbExtension->AtaFunction != ATA_FUNCTION_ATA_SMART) && (!IsNCQCommand(srbExtension)) && (slotContent->CmdHeader->PRDBC != RequestGetDataTransferLength(slotContent->Srb))) { if (slotContent->CmdHeader->PRDBC < RequestGetDataTransferLength(slotContent->Srb)) { // Buffer underrun, RequestSetDataTransferLength(slotContent->Srb, slotContent->CmdHeader->PRDBC); slotContent->Srb->SrbStatus = SrbStatus; } else { // Buffer overrun, return error NT_ASSERT(FALSE); slotContent->Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN; } } else { // If anything has set a STATUS on this SRB, honor that over the one passed in if (slotContent->Srb->SrbStatus == SRB_STATUS_PENDING) { slotContent->Srb->SrbStatus = SrbStatus; } } // 2.3 Monitor to see that any NCQ commands are completing if ((slotContent->Srb->SrbStatus == SRB_STATUS_SUCCESS) && (IsNCQCommand(srbExtension))) { ChannelExtension->StateFlags.NCQ_Succeeded = TRUE; } // 2.4 Give the slot back ReleaseSlottedCommand(ChannelExtension, i, AtDIRQL); // Request sense is handled here. if (LogExecuteFullDetail(adapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x10000046);//Completed one SRB } } } // 3.1 Start the next IO(s) if any ActivateQueue(ChannelExtension, AtDIRQL); return; } VOID SRBtoATA_CFIS( PAHCI_CHANNEL_EXTENSION ChannelExtension, PSLOT_CONTENT SlotContent ) /*++ Populates CFIS structure with an ATA command in the given slot It assumes: The slot is empty and not being used Called by: AhciHwStartIo It performs: (overview) 1 Fills in the CFIS structure (details) 1.1 Map SRB fields to CFIS fields 1.2 Special case mapping of NCQ Affected Variables/Registers: Command Table --*/ { PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(SlotContent->Srb); PAHCI_COMMAND_TABLE cmdTable = (PAHCI_COMMAND_TABLE)srbExtension; UNREFERENCED_PARAMETER(ChannelExtension); // 1.1 Map SRB fields to CFIS fields cmdTable->CFIS.FisType = 0x27; cmdTable->CFIS.PMPort = 0; // StorAHCI doesn't support Port Multiplier cmdTable->CFIS.Reserved1 = 0; cmdTable->CFIS.C = 1; cmdTable->CFIS.Command = srbExtension->TaskFile.Current.bCommandReg; // 1.2 Special case mapping of NCQ if (IsNCQCommand(srbExtension)) { cmdTable->CFIS.Features = srbExtension->TaskFile.Current.bSectorCountReg; cmdTable->CFIS.Features_Exp = srbExtension->TaskFile.Previous.bSectorCountReg; cmdTable->CFIS.SectorCount = (srbExtension->QueueTag << 3); cmdTable->CFIS.Dev_Head = 0xF & srbExtension->TaskFile.Current.bDriveHeadReg; cmdTable->CFIS.Dev_Head |= (1 << 6); if (SlotContent->StateFlags.FUA) { cmdTable->CFIS.Dev_Head |= ATA_NCQ_FUA_BIT; } else { cmdTable->CFIS.Dev_Head &= ~ATA_NCQ_FUA_BIT; } } else { cmdTable->CFIS.Features = srbExtension->TaskFile.Current.bFeaturesReg; cmdTable->CFIS.Features_Exp = srbExtension->TaskFile.Previous.bFeaturesReg; cmdTable->CFIS.SectorCount = srbExtension->TaskFile.Current.bSectorCountReg; cmdTable->CFIS.SectorCount_Exp = srbExtension->TaskFile.Previous.bSectorCountReg; cmdTable->CFIS.Dev_Head = srbExtension->TaskFile.Current.bDriveHeadReg; } // 1.1 Map SRB fields to CFIS fields cmdTable->CFIS.SectorNumber = srbExtension->TaskFile.Current.bSectorNumberReg; cmdTable->CFIS.SecNum_Exp = srbExtension->TaskFile.Previous.bSectorNumberReg; cmdTable->CFIS.CylLow = srbExtension->TaskFile.Current.bCylLowReg; cmdTable->CFIS.CylLow_Exp = srbExtension->TaskFile.Previous.bCylLowReg; cmdTable->CFIS.CylHigh = srbExtension->TaskFile.Current.bCylHighReg; cmdTable->CFIS.CylHigh_Exp = srbExtension->TaskFile.Previous.bCylHighReg; cmdTable->CFIS.ICC = 0; cmdTable->CFIS.Control = 0; // Device control consists of the 48bit HighOrderByte, SRST and nIEN. None apply here. cmdTable->CFIS.Auxiliary7_0 = 0; cmdTable->CFIS.Auxiliary15_8 = 0; cmdTable->CFIS.Auxiliary23_16 = 0; cmdTable->CFIS.Auxiliary31_24 = 0; } VOID SRBtoATAPI_CFIS( PAHCI_CHANNEL_EXTENSION ChannelExtension, PSLOT_CONTENT SlotContent ) /*++ Populates CFIS structure with an ATAPI command in the given slot It assumes: The slot is empty and not being used Called by: AhciHwStartIo It performs: (overview) 1 Populate the ACMD 2 Populate the PACKET command (details) 1.1 Memcopy CDB into ACMD 2.1 Put the PACKET Command in the CFIS 2.2 Populate DMA bit properly Affected Variables/Registers: Command Table --*/ { // These are used for copying a CDB to ACMD on ATAPI commands. ULONG dataLength; PVOID cdb; // 1.1 Memcopy CDB into ACMD PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(SlotContent->Srb); PAHCI_COMMAND_TABLE cmdTable = (PAHCI_COMMAND_TABLE)srbExtension; UNREFERENCED_PARAMETER(ChannelExtension); cdb = SrbGetCdb(SlotContent->Srb); dataLength = RequestGetDataTransferLength(SlotContent->Srb); if (cdb != NULL) { StorPortCopyMemory((PVOID)cmdTable->ACMD, cdb, 16); } else { NT_ASSERT(cdb != NULL); } // 2.1 Put the PACKET Command in the CFIS cmdTable->CFIS.FisType = 0x27; cmdTable->CFIS.PMPort = 0; // StorAHCI doesn't support Port Multiplier cmdTable->CFIS.Reserved1 = 0; cmdTable->CFIS.C = 1; cmdTable->CFIS.Command = IDE_COMMAND_ATAPI_PACKET; //A0 is the PACKET command // 2.2 Populate DMA bit properly, only use it for data transfer if (IsDmaCommand(srbExtension->Flags)) { cmdTable->CFIS.Features = 0x1; // If word 62 bit 15 is set to one, then the DMADIR bit in the PACKET command is required by the device for // PACKET commands using the DMA data transfer protocol and: // a) word 63 bits (2:0); -- MultiWordDMASupport & 0x7 == 0 // b) word 49 bit 15; -- Capabilities.InterleavedDmaSupported // c) word 49 bit 8; and -- Capabilities.DmaSupported // d) word 88 bits (6:0), -- UltraDMASupport & 0x7F == 0 //shall be cleared to zero. if ((ChannelExtension->DeviceExtension[0].IdentifyPacketData->DMADIR.DMADIRBitRequired != 0) && (ChannelExtension->DeviceExtension[0].IdentifyPacketData->Capabilities.DmaSupported != 0) && (ChannelExtension->DeviceExtension[0].IdentifyPacketData->Capabilities.InterleavedDmaSupported == 0) && ((ChannelExtension->DeviceExtension[0].IdentifyPacketData->MultiWordDMASupport & 0x7) == 0) && ((ChannelExtension->DeviceExtension[0].IdentifyPacketData->UltraDMASupport & 0x7F) == 0)) { // Bit 0 is DMA, bit 2 is DMADIR. (0 = transfer to the device; 4 = transfer to the host). cmdTable->CFIS.Features |= (srbExtension->Flags & ATA_FLAGS_DATA_IN) ? 4 : 0; } } else { cmdTable->CFIS.Features = 0x0; } // 2.1 Put the PACKET Command in the CFIS cmdTable->CFIS.SectorNumber = 0; cmdTable->CFIS.CylLow = (UCHAR)dataLength; //put the low byte of the in CylLow dataLength >>= 8; cmdTable->CFIS.CylHigh = (UCHAR)dataLength; //put the high byte in CylHigh cmdTable->CFIS.Dev_Head = 0; cmdTable->CFIS.SecNum_Exp = 0; cmdTable->CFIS.CylLow_Exp = 0; cmdTable->CFIS.CylHigh_Exp = 0; cmdTable->CFIS.Features_Exp = 0; cmdTable->CFIS.SectorCount = 0; cmdTable->CFIS.SectorCount_Exp = 0; cmdTable->CFIS.ICC = 0; cmdTable->CFIS.Control = 0; // Device control consists of the 48bit HighOrderByte, SRST and nIEN. None apply here. cmdTable->CFIS.Auxiliary7_0 = 0; cmdTable->CFIS.Auxiliary15_8 = 0; cmdTable->CFIS.Auxiliary23_16 = 0; cmdTable->CFIS.Auxiliary31_24 = 0; } VOID CfistoATA_CFIS( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ PSLOT_CONTENT SlotContent ) /*++ Populates CFIS structure with an ATA command in the given slot from a CFIS data structure. It assumes: The slot is empty and not being used Called by: AhciHwStartIo It performs: 1 Copy CFIS structure from SrbExtension 2 Make sure common data fields value are appropriate Affected Variables/Registers: Command Table --*/ { PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(SlotContent->Srb); PAHCI_COMMAND_TABLE cmdTable = (PAHCI_COMMAND_TABLE)srbExtension; UNREFERENCED_PARAMETER(ChannelExtension); // // Copy CFIS data structure. // StorPortCopyMemory(&cmdTable->CFIS, &srbExtension->Cfis, sizeof(AHCI_H2D_REGISTER_FIS)); // // When write a hiber file, the request can be retried. // Retry the command without Hybrid Hinting info. // if (IsDumpHiberMode(ChannelExtension->AdapterExtension) && IsNCQWriteCommand(srbExtension) && (srbExtension->RetryCount > 0) && (ChannelExtension->StateFlags.HybridInfoEnabledOnHiberFile == 0)) { cmdTable->CFIS.Auxiliary23_16 = 0; } // // Set common data fields. // if (IsNCQCommand(srbExtension)) { cmdTable->CFIS.Count7_0 = (srbExtension->QueueTag << 3); } cmdTable->CFIS.FisType = 0x27; cmdTable->CFIS.PMPort = 0; // StorAHCI doesn't support Port Multiplier cmdTable->CFIS.Reserved1 = 0; cmdTable->CFIS.C = 1; cmdTable->CFIS.Auxiliary15_8 = 0; cmdTable->CFIS.Auxiliary31_24 = 0; cmdTable->CFIS.ICC = 0; cmdTable->CFIS.Control = 0; // Device control consists of the 48bit HighOrderByte, SRST and nIEN. None apply here. } ULONG SRBtoPRDT( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ PSLOT_CONTENT SlotContent ) /*++ It assumes: MDLs and ScatterGatherList entries will not violate PRDT rules Called by: AhciHwStartIo It performs: (overview) 1 Get the DataBuffer's address 2 Map SGL entries into PRDT entries (details) 1.1 Get the ScatterGatherList 1.2 Verify that the DataBuffer is properly aligned 2.1 Map SGL entries into PRDT entries 2.2 Break up all 128K single entry IO into 2 64K IO entries 2.3 Verify that the DataLength is even Affected Variables/Registers: Return Values: The number of entries generated to fill the PRDT If the value returned is -1 the PRDT could not be built. --*/ { ULONG i; PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(SlotContent->Srb); PAHCI_COMMAND_TABLE cmdTable = (PAHCI_COMMAND_TABLE)srbExtension; PLOCAL_SCATTER_GATHER_LIST sgl = srbExtension->Sgl; if (sgl == NULL) { // Return as invalid request in case of cannot get scatter gather list. NT_ASSERT(FALSE); return (ULONG)-1; } for (i = 0; i < sgl->NumberOfElements; i++) { // 1.2 Verify that the DataBuffer is properly aligned if ((sgl->List[i].PhysicalAddress.LowPart & 0x1) == 0) { // 2.1 Map SGL entries into PRDT entries if (sgl->List[i].Length != 0x20000) { cmdTable->PRDT[i].DBA.AsUlong = sgl->List[i].PhysicalAddress.LowPart; if (ChannelExtension->AdapterExtension->CAP.S64A) {//If the controller supports 64 bits, write the high part too cmdTable->PRDT[i].DBAU = sgl->List[i].PhysicalAddress.HighPart; } // 2.2 Break up a 128K single entry IO into 2 64K IO entries (128K is max transfer so there can be only 1 in any SGL) // although one entry can represent at max 4M length IO, some adapters cannot handle a DBC >= 128K. } else { // Entry 0 cmdTable->PRDT[0].DBA.AsUlong = sgl->List[0].PhysicalAddress.LowPart; if (ChannelExtension->AdapterExtension->CAP.S64A) {//If the controller supports 64 bits, write the high part too cmdTable->PRDT[0].DBAU = sgl->List[0].PhysicalAddress.HighPart; } cmdTable->PRDT[0].DI.DBC = (0x10000 - 1); // Entry 1 cmdTable->PRDT[1].DBA.AsUlong = (sgl->List[0].PhysicalAddress.LowPart + 0x10000); if (ChannelExtension->AdapterExtension->CAP.S64A) {//If the controller supports 64 bits, write the high part too if ((sgl->List[0].PhysicalAddress.LowPart + 0x10000) < sgl->List[0].PhysicalAddress.LowPart) { cmdTable->PRDT[1].DBAU = (sgl->List[0].PhysicalAddress.HighPart + 1); //add 1 to the highpart if adding 0x10000 caused a rollover } else { cmdTable->PRDT[1].DBAU = sgl->List[0].PhysicalAddress.HighPart; } } cmdTable->PRDT[1].DI.DBC = (0x10000 - 1); return 2; } } else { NT_ASSERT(FALSE); //Shall Not Pass return (ULONG)-1; } // 1.3 Verify that the DataLength is even // all SATA transfers must be even // DBC is a 0 based number (i.e. 0 is 1, 1 is 2, etc. // sgl->Elements.Length is not (i.e. 0 is 0, 1 is 1, etc. if ((sgl->List[i].Length & 1) == 0) { //therefore length must be even here // 2.3 Set Datalength in the PRDT entries cmdTable->PRDT[i].DI.DBC = sgl->List[i].Length - 1; //but it must be odd here } else if (sgl->List[i].Length <= RequestGetDataTransferLength(SlotContent->Srb)) { // Storport may send down SCSI commands with odd number of data transfer length, and it builds SGL using that transfer length value. // we use the length -1 to get as much data as we can. If the data length is over (length - 1), buffer overrun will be reported when the command is completed. RequestSetDataTransferLength(SlotContent->Srb, RequestGetDataTransferLength(SlotContent->Srb) - 1); cmdTable->PRDT[i].DI.DBC = sgl->List[i].Length - 2; } else { NT_ASSERT(FALSE); //Shall Not Pass return (ULONG)-1; } } return sgl->NumberOfElements; } VOID SRBtoCmdHeader( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ PSLOT_CONTENT SlotContent, _In_ ULONG Length, _In_ BOOLEAN Reset ) /*++ It assumes: The chosen slot is available and not in use Called by: AhciHwStartIo It performs: Steps defined in AHCI 1.2 section 5.5.1 step #3 Affected Variables/Registers: Command Header --*/ { PAHCI_COMMAND_HEADER cmdHeader = SlotContent->CmdHeader; PSTORAGE_REQUEST_BLOCK srb = SlotContent->Srb; PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(srb); UNREFERENCED_PARAMETER(ChannelExtension); // a. PRDTL containing the number of entries in the PRD table cmdHeader->DI.PRDTL = Length; // b. CFL set to the length of the command in the CFIS area cmdHeader->DI.CFL = 5; // c. A bit set if it is an ATAPI command cmdHeader->DI.A = (srbExtension->AtaFunction & ATA_FUNCTION_ATAPI_COMMAND) ? 1 : 0; // d. W (Write) bit set if data is going to the device cmdHeader->DI.W = (srbExtension->Flags & ATA_FLAGS_DATA_OUT) ? 1 : 0; // e. P (Prefetch) bit optionally set (see rules in section 5.5.2) //Some controllers have problems if P is set. cmdHeader->DI.P = 0; // f. If a Port Multiplier is attached, the PMP field set to the correct Port Multiplier port. cmdHeader->DI.PMP = 0; // Reset cmdHeader->DI.R = Reset; cmdHeader->DI.B = 0; cmdHeader->DI.C = Reset; // Initialize the PRD byte count cmdHeader->PRDBC = 0; cmdHeader->Reserved[0] = 0; cmdHeader->Reserved[1] = 0; cmdHeader->Reserved[2] = 0; cmdHeader->Reserved[3] = 0; } BOOLEAN AhciProcessIo( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ PSTORAGE_REQUEST_BLOCK Srb, _In_ BOOLEAN AtDIRQL ) /* This routine does following: 1. Check if no device command associated with Srb, or the port failed to start. Bail out in these cases. 2. Central place for special command handling. 3. Get available slot 4. Fill the slot; Program command Table & Header; Put IO in Slice Note: This routine can be called even the Port is stopped. This routine does NOT program IO to adapter. */ { PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(Srb); UCHAR pathId = 0; UCHAR targetId = 0; UCHAR lun = 0; STOR_LOCK_HANDLE lockHandle = { InterruptLock, 0 }; SrbGetPathTargetLun(Srb, &pathId, &targetId, &lun); // 1.0 complete Srb if no command should be sent to device. if (srbExtension->AtaFunction == 0) { NT_ASSERT(FALSE); // should investigate if this ASSERT fires Srb->SrbStatus = SRB_STATUS_ERROR; MarkSrbToBeCompleted(Srb); AhciCompleteRequest(ChannelExtension, Srb, AtDIRQL); return TRUE; } // 1.1 Check if command processing should happen if (ChannelExtension->StartState.ChannelNextStartState == StartFailed) { Srb->SrbStatus = SRB_STATUS_NO_DEVICE; MarkSrbToBeCompleted(Srb); AhciCompleteRequest(ChannelExtension, Srb, AtDIRQL); return TRUE; } // 2. central place for command special handling. this part cannot be put in command translation layer, as there might be internal command also needs special handling. // 2.1 central place for special handling of Enable/Disable WRITE CACHE, update persistent settings command. if ((srbExtension->AtaFunction == ATA_FUNCTION_ATA_COMMAND) && (srbExtension->TaskFile.Current.bCommandReg == IDE_COMMAND_SET_FEATURE)) { if (srbExtension->TaskFile.Current.bFeaturesReg == IDE_FEATURE_DISABLE_WRITE_CACHE) { // IDE_FEATURE_DISABLE_WRITE_CACHE => Modify Persistent Settings UpdateSetFeatureCommands(ChannelExtension, IDE_FEATURE_ENABLE_WRITE_CACHE, IDE_FEATURE_DISABLE_WRITE_CACHE, 0, 0); } if (srbExtension->TaskFile.Current.bFeaturesReg == IDE_FEATURE_ENABLE_WRITE_CACHE) { // IDE_FEATURE_ENABLE_WRITE_CACHE => Modify Persistent Settings UpdateSetFeatureCommands(ChannelExtension, IDE_FEATURE_DISABLE_WRITE_CACHE, IDE_FEATURE_ENABLE_WRITE_CACHE, 0, 0); } } // 2.2 update the Identify command for enumeration. This needs to be here waiting for PxSIG to be accessible. if ((srbExtension->AtaFunction == ATA_FUNCTION_ATA_IDENTIFY) && (srbExtension->CompletionRoutine == AhciPortIdentifyDevice) && (srbExtension->TaskFile.Current.bCommandReg == IDE_COMMAND_NOT_VALID)) { // For enumeration command, it's safe now to check the PxSIG register as starting port has been executed. // Note: there is chance that the signature register is FFFFFFFFh when accessing it here because reset is in progress. // Handle this special case in the follow checking device type code. ULONG sig = 0; sig = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SIG.AsUlong); if (sig == ATA_DEVICE_SIGNATURE_ATA) { srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_IDENTIFY; ChannelExtension->DeviceExtension[0].DeviceParameters.AtaDeviceType = DeviceIsAta; } else if (sig == ATA_DEVICE_SIGNATURE_ATAPI) { srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_ATAPI_IDENTIFY; ChannelExtension->DeviceExtension[0].DeviceParameters.AtaDeviceType = DeviceIsAtapi; } else { ULONG ssts = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SSTS.AsUlong); RecordExecutionHistory(ChannelExtension, 0x10090020); // Unexpected signature register value if ((sig == 0) || ((sig == (ULONG)~0) && (ssts != (ULONG)~0))) { // Case 1: A value of 0 is not a valid signature. Some devices // initially report a valid signature but soon after the PxSIG // register becomes 0. // Case 2: A value of FFFFFFFFh is set to signature register during reset according to AHCI 1.3.1 spec, // while the value maybe also FFFFFFFFh when device is removed, because PCI configuration space that mapped with AHCI registers // are FFFFFFFFh. To differentiate whether it is reset or device removed, check Px.SSTS register value(it is 0 during reset). // // If above situations happen, just set the identify command according to the currently known device type. switch (ChannelExtension->DeviceExtension[0].DeviceParameters.AtaDeviceType) { case DeviceIsAta: srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_IDENTIFY; break; case DeviceIsAtapi: default: srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_ATAPI_IDENTIFY; break; } } else { // Other devices types, such as with signature ATA_DEVICE_SIGNATURE_ENCLOSURE, or // ATA_DEVICE_SIGNATURE_PORT_MULTIPLIER, are not supported. // IDENTIFY PACKET DEVICE command will fail on these devices. srbExtension->TaskFile.Current.bCommandReg = IDE_COMMAND_ATAPI_IDENTIFY; ChannelExtension->DeviceExtension[0].DeviceParameters.AtaDeviceType = DeviceUnknown; } } } // 2.3 this is central place to update this register value in case it's wrongly set. if (IsAtaCommand(srbExtension->AtaFunction)) { SetDeviceReg((&srbExtension->TaskFile.Current), 0); //make sure set to device0 } // 3 Find an available slot/tag (AHCI 1.1 Section 5.5.1) if (!AtDIRQL) { AhciInterruptSpinlockAcquire(ChannelExtension->AdapterExtension, ChannelExtension->PortNumber, &lockHandle); } GetAvailableSlot(ChannelExtension, Srb); //srbExtension->QueueTag will be set // 3.1 If no tag is available, reject the command to be retried later if (srbExtension->QueueTag > ChannelExtension->AdapterExtension->CAP.NCS) { // Wait for 8 IO (random picked number) being completed before re-starting sending IO to miniport for this device. // if filled up, if (!IsMiniportInternalSrb(ChannelExtension, Srb)) { // Report busy only for IOs from Storport StorPortDeviceBusy(ChannelExtension->AdapterExtension, pathId, targetId, lun, min(8, ChannelExtension->AdapterExtension->CAP.NCS)); } Srb->SrbStatus = SRB_STATUS_BUSY; MarkSrbToBeCompleted(Srb); AhciCompleteRequest(ChannelExtension, Srb, TRUE); RecordExecutionHistory(ChannelExtension, 0x10030020); //AllocateQueueTagFailed goto exit; } #ifdef DBG { ULONG ci; ULONG sact; ci = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CI); sact = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SACT); // 3.2 if the tag returned can't be used, something is seriously wrong. Reject the command to be retried later if ((( (1 << srbExtension->QueueTag) & ci) > 0) || (((1 << srbExtension->QueueTag) & sact) > 0)) { if (!IsMiniportInternalSrb(ChannelExtension, Srb)) { // pause device only for IOs from Storport StorPortPauseDevice(ChannelExtension->AdapterExtension, pathId, targetId, lun, 1); //pause for 1 seconds. } // One reason why this can happen is if the adapter was removed. if (IsAdapterRemoved(ChannelExtension->AdapterExtension)) { Srb->SrbStatus = SRB_STATUS_NO_DEVICE; } else { NT_ASSERT(FALSE); //can this happen? catch it. Srb->SrbStatus = SRB_STATUS_BUSY; } MarkSrbToBeCompleted(Srb); AhciCompleteRequest(ChannelExtension, Srb, TRUE); RecordExecutionHistory(ChannelExtension, 0x10040020);//Tag given for slot in use goto exit; } } #endif //srbExtension->QueueTag and Slot[srbExtension->QueueTag] are now guaranteed ready. AhciFormIo (ChannelExtension, Srb, TRUE); exit: if (!AtDIRQL) { AhciInterruptSpinlockRelease(ChannelExtension->AdapterExtension, ChannelExtension->PortNumber, &lockHandle); } return TRUE; } BOOLEAN AhciFormIo( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ PSTORAGE_REQUEST_BLOCK Srb, _In_ BOOLEAN AtDIRQL ) /* Caller: only AhciProcessIo() */ { PAHCI_SRB_EXTENSION srbExtension; PSLOT_CONTENT slotContent; PAHCI_COMMAND_TABLE cmdTable; PAHCI_COMMAND_HEADER cmdHeader; STOR_PHYSICAL_ADDRESS cmdTablePhysicalAddress; ULONG prdtLength = 0; PCDB cdb = SrbGetCdb(Srb); srbExtension = GetSrbExtension(Srb); // Command header is allocated in ChannelExtension for all command slots, get the correct one. cmdHeader = ChannelExtension->CommandList; cmdHeader += srbExtension->QueueTag; slotContent = &ChannelExtension->Slot[srbExtension->QueueTag]; // 1. setup slot content and log history slotContent->Srb = Srb; slotContent->CmdHeader = cmdHeader; // 1.1 Update FUA tracking flag if needed if (!IsReturnResults(srbExtension->Flags) && (cdb != NULL) && ((cdb->CDB10.OperationCode == SCSIOP_WRITE) || (cdb->CDB10.OperationCode == SCSIOP_WRITE16)) && (cdb->CDB10.ForceUnitAccess == 1) && IsFuaSupported(ChannelExtension)) { // Keep track of FUA to add it back in when the command is put in the FIS slotContent->StateFlags.FUA = TRUE; } if (LogCommand(ChannelExtension->AdapterExtension->LogFlags)) { // 1/2 Log command starting part for our debugging records PCOMMAND_HISTORY cmdHistory; ChannelExtension->CommandHistoryNextAvailableIndex++; ChannelExtension->CommandHistoryNextAvailableIndex %= 64; slotContent->CommandHistoryIndex = ChannelExtension->CommandHistoryNextAvailableIndex; cmdHistory = &ChannelExtension->CommandHistory[slotContent->CommandHistoryIndex]; cmdHistory->Tag = srbExtension->QueueTag; StorPortCopyMemory(&cmdHistory->InitialTaskFile, &srbExtension->TaskFile, 0x10); cmdHistory->InitialPx[0] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CLB.AsUlong); cmdHistory->InitialPx[1] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CLBU); cmdHistory->InitialPx[2] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->FB.AsUlong); cmdHistory->InitialPx[3] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->FBU); cmdHistory->InitialPx[4] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong); cmdHistory->InitialPx[5] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IE.AsUlong); cmdHistory->InitialPx[6] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CMD.AsUlong); cmdHistory->InitialPx[7] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->DW7_Reserved); cmdHistory->InitialPx[8] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->TFD.AsUlong); cmdHistory->InitialPx[9] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SIG.AsUlong); cmdHistory->InitialPx[10] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SSTS.AsUlong); cmdHistory->InitialPx[11] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SCTL.AsUlong); cmdHistory->InitialPx[12] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong); cmdHistory->InitialPx[13] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SACT); cmdHistory->InitialPx[14] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CI); cmdHistory->InitialPx[15] = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SNTF.AsUlong); cmdHistory->Function = SrbGetSrbFunction(slotContent->Srb); cmdHistory->SrbStatus = 0; } // Already get a slot, after this point, any request completion effort needs to release the slot. // just like what's done in: ReleaseSlottedCommand() // 2. Program the CFIS in the CommandTable (allocated in srbExtension) cmdTable = (PAHCI_COMMAND_TABLE)srbExtension; if (IsAtapiCommand(srbExtension->AtaFunction)) { SRBtoATAPI_CFIS(ChannelExtension, slotContent); } else if (IsAtaCfisPayload(srbExtension->AtaFunction)) { CfistoATA_CFIS(ChannelExtension, slotContent); } else if (IsAtaCommand(srbExtension->AtaFunction)) { SRBtoATA_CFIS(ChannelExtension, slotContent); } else { NT_ASSERT(FALSE); slotContent->Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST; AhciCompleteJustSlottedRequest(ChannelExtension, Srb, AtDIRQL); RecordExecutionHistory(ChannelExtension, 0x10070020); //Invalid ATA Function return TRUE; } // 3. Build the PRD Table in CommandTable. if (IsDataTransferNeeded(slotContent->Srb)) { prdtLength = SRBtoPRDT(ChannelExtension, slotContent); if (prdtLength == -1) { NT_ASSERT(FALSE); slotContent->Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST; AhciCompleteJustSlottedRequest(ChannelExtension, Srb, AtDIRQL); RecordExecutionHistory(ChannelExtension, 0x10060020); //Invalid SGL return TRUE; } } // 4. Program the Command Header (allocated in ChannelExtension for all command slots) SRBtoCmdHeader(ChannelExtension, slotContent, prdtLength, FALSE); // 4.1. Get the Command Table's physical address to verify the alignment and program cmdHeader->CTBA if ((PSTORAGE_REQUEST_BLOCK)&ChannelExtension->Local.Srb == Srb) { cmdTablePhysicalAddress = ChannelExtension->Local.SrbExtensionPhysicalAddress; } else if ((PSTORAGE_REQUEST_BLOCK)&ChannelExtension->Sense.Srb == Srb) { cmdTablePhysicalAddress = ChannelExtension->Sense.SrbExtensionPhysicalAddress; } else { ULONG length; // SRB is needed for StorPortGetPhysicalAddress to calculate logical address when DMAR is enabled. cmdTablePhysicalAddress = StorPortGetPhysicalAddress(ChannelExtension->AdapterExtension, (PSCSI_REQUEST_BLOCK)Srb, srbExtension, &length); } if ((cmdTablePhysicalAddress.LowPart % 128) == 0) { cmdHeader->CTBA.AsUlong = cmdTablePhysicalAddress.LowPart; } else { slotContent->Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST; AhciCompleteJustSlottedRequest(ChannelExtension, Srb, AtDIRQL); RecordExecutionHistory(ChannelExtension, 0x10080020);//Invalid PhyscialAddress alignment return TRUE; } // If the controller supports 64 bits, write the high part too if (ChannelExtension->AdapterExtension->CAP.S64A) { cmdHeader->CTBAU = cmdTablePhysicalAddress.HighPart; } // 5. Sort it, put request in one of IO Slices. // 5.1. Keep track of high priority outside of the Slices if (IsHighPriorityCommand(srbExtension->Flags)) { ChannelExtension->SlotManager.HighPriorityAttribute |= ( 1 << srbExtension->QueueTag); } // 5.2. Put the slot content in the correct Slice if (IsNCQCommand(srbExtension)) { ChannelExtension->SlotManager.NCQueueSlice |= ( 1 << srbExtension->QueueTag); } else if (IsReturnResults(srbExtension->Flags)) { ChannelExtension->SlotManager.SingleIoSlice |= ( 1 << srbExtension->QueueTag); } else if (IsNormalCommand(slotContent->Srb)) { ChannelExtension->SlotManager.NormalQueueSlice |= ( 1 << srbExtension->QueueTag); } else { ChannelExtension->SlotManager.SingleIoSlice |= ( 1 << srbExtension->QueueTag); } if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) { RecordExecutionHistory(ChannelExtension, 0x10000020);//Exit AhciHwStartIo } return TRUE; } PSCSI_REQUEST_BLOCK BuildRequestSenseSrb( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ PSTORAGE_REQUEST_BLOCK FailingSrb ) { STOR_PHYSICAL_ADDRESS dataBufferPhysicalAddress; ULONG length; PCDB cdb; PSCSI_REQUEST_BLOCK senseSrb = &ChannelExtension->Sense.Srb; PAHCI_SRB_EXTENSION srbExtension = ChannelExtension->Sense.SrbExtension; PVOID srbSenseBuffer = NULL; UCHAR srbSenseBufferLength = 0; RequestGetSrbScsiData(FailingSrb, NULL, NULL, &srbSenseBuffer, &srbSenseBufferLength); // 1. Validate if senseSrb should be used. if ((srbExtension == NULL) || (srbExtension->AtaFunction != 0) || (srbSenseBuffer == NULL) || (srbSenseBufferLength == 0)) { // Request Sense is for ATAPI commands which are single IOs. At the same time there should be only one command failed and needs this Srb. NT_ASSERT(srbExtension && srbExtension->AtaFunction == 0); return NULL; } if ((srbSenseBuffer != NULL) && (srbSenseBufferLength > 0)) { AhciZeroMemory((PCHAR)srbSenseBuffer, srbSenseBufferLength); } // 2. Initialize Srb and SrbExtension structures. AhciZeroMemory((PCHAR)senseSrb, sizeof(SCSI_REQUEST_BLOCK)); AhciZeroMemory((PCHAR)srbExtension, sizeof(AHCI_SRB_EXTENSION)); // 3. Setup Srb and CDB. Note that Sense Srb uses SCSI_REQUEST_BLOCK type. senseSrb->Length = sizeof(SCSI_REQUEST_BLOCK); senseSrb->Function = SRB_FUNCTION_EXECUTE_SCSI; senseSrb->PathId = (UCHAR)ChannelExtension->PortNumber; senseSrb->SrbFlags = SRB_FLAGS_DATA_IN; senseSrb->DataBuffer = srbSenseBuffer; senseSrb->DataTransferLength = srbSenseBufferLength; senseSrb->SrbExtension = (PVOID)srbExtension; senseSrb->OriginalRequest = (PVOID)FailingSrb; senseSrb->TimeOutValue = 1; senseSrb->CdbLength = 6; cdb = (PCDB)&senseSrb->Cdb; cdb->CDB6INQUIRY.OperationCode = SCSIOP_REQUEST_SENSE; cdb->CDB6INQUIRY.AllocationLength = (UCHAR)senseSrb->DataTransferLength; // 4. Fill in the srbExtension fields srbExtension->AtaFunction = ATA_FUNCTION_REQUEST_SENSE; srbExtension->Flags = ATA_FLAGS_DATA_IN; // SRB is needed for StorPortGetPhysicalAddress to calculate logical address when DMAR is enabled. dataBufferPhysicalAddress = StorPortGetPhysicalAddress(ChannelExtension->AdapterExtension, (PSCSI_REQUEST_BLOCK)FailingSrb, srbSenseBuffer, &length); if (dataBufferPhysicalAddress.QuadPart == 0) { NT_ASSERT(FALSE); //Shall Not Pass return NULL; } srbExtension->LocalSgl.NumberOfElements = 1; srbExtension->LocalSgl.List[0].PhysicalAddress.LowPart = dataBufferPhysicalAddress.LowPart; srbExtension->LocalSgl.List[0].PhysicalAddress.HighPart = dataBufferPhysicalAddress.HighPart; srbExtension->LocalSgl.List[0].Length = senseSrb->DataTransferLength; srbExtension->Sgl = &srbExtension->LocalSgl; return senseSrb; } VOID AhciPortFailAllIos( _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension, _In_ UCHAR SrbStatus, _In_ BOOLEAN AtDIRQL ) { UCHAR i; // Complete all requests in slots for (i = 0; i <= ChannelExtension->AdapterExtension->CAP.NCS; i++) { if (ChannelExtension->Slot[i].Srb != NULL) { ChannelExtension->Slot[i].Srb->SrbStatus = SrbStatus; AhciCompleteJustSlottedRequest(ChannelExtension, ChannelExtension->Slot[i].Srb, AtDIRQL); } } // Requests in completion queue will be completed by AhciPortSrbCompletionDpcRoutine. return; } VOID AhciPortSrbCompletionDpcRoutine( _In_ PSTOR_DPC Dpc, _In_ PVOID AdapterExtension, _In_opt_ PVOID SystemArgument1, _In_opt_ PVOID SystemArgument2 ) { PAHCI_CHANNEL_EXTENSION channelExtension = (PAHCI_CHANNEL_EXTENSION)SystemArgument1; STOR_LOCK_HANDLE lockhandle = { InterruptLock, 0 }; PSTORAGE_REQUEST_BLOCK srb = NULL; PSRB_COMPLETION_ROUTINE completionRoutine = NULL; BOOLEAN reservedSlotInUse = FALSE; BOOLEAN sendCommand = FALSE; BOOLEAN sendCommandForLocalSrb = FALSE; ULONGLONG count = 0; // record the number of Srb being completed in this routine. It's for diagnostic purpose. UNREFERENCED_PARAMETER(Dpc); UNREFERENCED_PARAMETER(SystemArgument2); if (channelExtension == NULL) { NT_ASSERT(channelExtension != NULL); return; } if (LogExecuteFullDetail(channelExtension->AdapterExtension->LogFlags)) { RecordExecutionHistory(channelExtension, 0x0000001e);//Enter AhciPortSrbCompletionDpcRoutine } StorPortQuerySystemTime(&(channelExtension->LastTimeStampDpcStart)); // // Preserve the use status about Reserved Slot. // reservedSlotInUse = (channelExtension->StateFlags.ReservedSlotInUse == 1); do { AhciInterruptSpinlockAcquire(channelExtension->AdapterExtension, channelExtension->PortNumber, &lockhandle); srb = RemoveQueue(channelExtension, &channelExtension->CompletionQueue, 0xDEADC0DE, 0x9F); AhciInterruptSpinlockRelease(channelExtension->AdapterExtension, channelExtension->PortNumber, &lockhandle); if (srb != NULL) { PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension(srb); BOOLEAN completeSrb = TRUE; completionRoutine = srbExtension->CompletionRoutine; srbExtension->AtaFunction = 0; // clear this field. srbExtension->CompletionRoutine = NULL; if (completionRoutine != NULL) { completionRoutine(channelExtension, srb); // A request with STATUS_BUS_RESET should be completed after running completion routine. if ((srbExtension->AtaFunction != 0) && (!SrbShouldBeCompleted(srbExtension->Flags)) && (srb->SrbStatus != SRB_STATUS_BUS_RESET)) { // new command associated needs to be processed, do not complete the request. AhciProcessIo(channelExtension, srb, FALSE); // This Srb should not be completed yet completeSrb = FALSE; sendCommand = TRUE; if (srb == (PSTORAGE_REQUEST_BLOCK)&channelExtension->Local.Srb) { sendCommandForLocalSrb = TRUE; } } else if (SrbShouldBeCompleted(srbExtension->Flags)) { // Clear the flag CLRMASK(srbExtension->Flags, ATA_FLAGS_COMPLETE_SRB); } } else { // An Srb without completion routine should be completed. } if ((srb == (PSTORAGE_REQUEST_BLOCK)&channelExtension->Local.Srb) && reservedSlotInUse && channelExtension->StateFlags.ReservedSlotInUse && (!sendCommandForLocalSrb)) { // // This should never happen. ReservedSlotInUse flag will stuck. // NT_ASSERT(FALSE); RecordExecutionHistory(channelExtension, 0x1001001e); //AhciPortSrbCompletionDpcRoutine, ReservedSlotInUse starts to stuck AhciTelemetryLog(channelExtension->AdapterExtension, (PSTOR_ADDRESS)&(channelExtension->DeviceExtension->DeviceAddress), AHCI_TELEMETRY_DRIVER_VERSION, AhciTelemetryEventIdReservedSlotStuck, "Reserved Slot Stuck", AHCI_TELEMETRY_EVENT_VERSION, 0, 0, NULL, "CommandsIssued", channelExtension->SlotManager.CommandsIssued, "NCQueueSlice", channelExtension->SlotManager.NCQueueSlice, "NormalQueueSlice", channelExtension->SlotManager.NormalQueueSlice, "SingleIoSlice", channelExtension->SlotManager.SingleIoSlice, "CommandsToComplete", channelExtension->SlotManager.CommandsToComplete, "StateFlags", *(ULONGLONG *)&(channelExtension->StateFlags), "StartState", *(ULONGLONG *)&(channelExtension->StartState), "Flags|Function|Status", (ULONGLONG)((((ULONGLONG)srbExtension->Flags) << 32) | (((ULONGLONG)srbExtension->AtaFunction) << 16) | (ULONGLONG)(srb->SrbStatus)) ); } if (completeSrb) { // Release active reference for port/device and adapter. if ((srbExtension->Flags & ATA_FLAGS_ACTIVE_REFERENCE) != 0) { PortReleaseActiveReference(channelExtension, srb); } if (!IsMiniportInternalSrb(channelExtension, srb)) { if (SRB_STATUS(srb->SrbStatus) != SRB_STATUS_SUCCESS) { } NT_ASSERT(srb->SrbStatus != SRB_STATUS_PENDING); StorPortNotification(RequestComplete, AdapterExtension, srb); } } } count++; } while (srb != NULL); // // If Reserved Slot is freed by one of completion routine, send pending commands to device. // if (reservedSlotInUse && (channelExtension->StateFlags.ReservedSlotInUse == 0)) { sendCommand = TRUE; } if (sendCommand) { ActivateQueue(channelExtension, (IsDumpMode(channelExtension->AdapterExtension) ? (TRUE) : (FALSE))); } StorPortQuerySystemTime(&(channelExtension->LastTimeStampDpcCompletion)); if (LogExecuteFullDetail(channelExtension->AdapterExtension->LogFlags)) { RecordExecutionHistory(channelExtension, 0x1000001e);//Exit AhciPortSrbCompletionDpcRoutine } return; } #if _MSC_VER >= 1200 #pragma warning(pop) #else #pragma warning(default:4214) #pragma warning(default:4201) #endif