path: root/storage/miniports/storahci/src/entrypts.c

/*++

Copyright (C) Microsoft Corporation, 2009

Module Name:

    entrypts.c

Abstract:

    This file contains function of entry points to the AHCI miniport.


Notes:

Revision History:

--*/

#if _MSC_VER >= 1200
#pragma warning(push)
#endif

#pragma warning(disable:4152) // nonstandard extension, function/data pointer conversion in expression
#pragma warning(disable:4214) // bit field types other than int
#pragma warning(disable:4201) // nameless struct/union

#include "generic.h"


// storahci.sys global variables
PVOID   g_AdapterExtension[4] = {0};
UCHAR   g_AdapterExtensionIndex = 0;

//
// Boot device telemetry default initialization values, override by the registry table
//

ULONG
AhciPublicGPLogTableAddresses[TC_PUBLIC_DEVICEDUMP_CONTENT_GPLOG_MAX] =
    {
        //0x01,     // Summary SMART error log. Accessed by SMART command only
        //0x02,     // Comprehensive SMART error log. Accessed by SMART command only
        0x04,       // Device statistics page
        0x08,       // Power conditions
        0x0d,       // LPS Mis-alignment
        0x10,       // NCQ command error log
        0x11,       // SATA PHY event counters log
        0
    };

ULONG
AhciGPLogPageIntoPrivate =  IDE_GP_LOG_CURRENT_DEVICE_INTERNAL_STATUS;

//

ULONG
DriverEntry(
    _In_ PVOID Argument1,    //IN PDRIVER_OBJECT  DriverObject,
    _In_ PVOID Argument2     //IN PUNICODE_STRING  RegistryPath
    )

/*++

Routine Description:

    Initial entry point for miniport driver.

Arguments:

    Driver Object

Return Value:

    Status from StorPortInitialize()

--*/

{
    ULONG status;
    HW_INITIALIZATION_DATA hwInitializationData = {0};

    DebugPrint((1, "\tSTORAHCI, Storport AHCI Miniport Driver.\n"));

    hwInitializationData.HwInitializationDataSize = sizeof(HW_INITIALIZATION_DATA);

    // required miniport entry point routines.
    hwInitializationData.HwInitialize = AhciHwInitialize;
    hwInitializationData.HwStartIo = AhciHwStartIo;
    hwInitializationData.HwInterrupt = AhciHwInterrupt;
    hwInitializationData.HwFindAdapter = AhciHwFindAdapter;
    hwInitializationData.HwResetBus = AhciHwResetBus;
    hwInitializationData.HwAdapterControl = AhciHwAdapterControl;
    hwInitializationData.HwBuildIo = AhciHwBuildIo;
    hwInitializationData.HwTracingEnabled = AhciHwTracingEnabled;
    hwInitializationData.HwUnitControl = AhciHwUnitControl;

    // Specify adapter specific information.
    hwInitializationData.AutoRequestSense = TRUE;
    hwInitializationData.NeedPhysicalAddresses = TRUE;
    hwInitializationData.NumberOfAccessRanges = NUM_ACCESS_RANGES;
    
    //
    // Support both PCI/ACPI enumerations on ARM64 platform for SATA device.
    // Miniport uses flag to indicate storport to query/override the interface type reported here.
    //
#if defined(_ARM_) || defined(_ARM64_)
    hwInitializationData.FeatureSupport |= STOR_FEATURE_SET_ADAPTER_INTERFACE_TYPE;
    hwInitializationData.AdapterInterfaceType = InterfaceTypeUndefined;
#else
    hwInitializationData.AdapterInterfaceType = PCIBus;
#endif
    hwInitializationData.MapBuffers = STOR_MAP_NON_READ_WRITE_BUFFERS;
    hwInitializationData.TaggedQueuing = TRUE;
    hwInitializationData.MultipleRequestPerLu = TRUE;
    hwInitializationData.FeatureSupport |= STOR_FEATURE_ATA_PASS_THROUGH;                       // indicating this miniport driver supports ATA PASS-THROUGH(16) command.
    hwInitializationData.FeatureSupport |= STOR_FEATURE_FULL_PNP_DEVICE_CAPABILITIES;           // indicating this miniport driver supplies values for all PnP Device Capability fields.
    hwInitializationData.FeatureSupport |= STOR_FEATURE_DUMP_POINTERS;                          // indicating this miniport support the dump pointers SRBs
    hwInitializationData.FeatureSupport |= STOR_FEATURE_DUMP_RESUME_CAPABLE;                    // indicating this miniport driver supports resume capability for dump stack.
    hwInitializationData.FeatureSupport |= STOR_FEATURE_DEVICE_NAME_NO_SUFFIX;                  // indicating the miniport driver prefers device friendly name without suffix: "SCSI <type> Device"
    hwInitializationData.FeatureSupport |= STOR_FEATURE_DEVICE_DESCRIPTOR_FROM_ATA_INFO_VPD;    // indicating that port driver forms STORAGE_DEVICE_DESCRIPTOR from ATA Information VPD page rather than INQUIRY data
    hwInitializationData.FeatureSupport |= STOR_FEATURE_EXTRA_IO_INFORMATION;                   // Indicating that miniport driver wants SRBEX_DATA_IO_INFO in a SRBEX if available
    hwInitializationData.FeatureSupport |= STOR_FEATURE_ADAPTER_NOT_REQUIRE_IO_PORT;            // Indicating that miniport driver doesn't require IO Port resource for its adapter.
    hwInitializationData.FeatureSupport |= STOR_FEATURE_DUMP_INFO;                              // Indicating that the miniport driver supports the dump info SRBs.

    // Set required extension sizes.
    hwInitializationData.DeviceExtensionSize = sizeof(AHCI_ADAPTER_EXTENSION);

    // NOTE: Command Table (1st field in AHCI_SRB_EXTENSION structure) must align to 128 bytes as physical limitation.
    // StorPort does not have interface allowing miniport requiring this.
    // Adding 128 in SrbExtensionSize so that we can use the part starting from right alignment.
    hwInitializationData.SrbExtensionSize = sizeof(AHCI_SRB_EXTENSION) + 128 ; // SrbExtension contains AHCI_SRB_EXTENSION

    //
    // Support SrbEx by default.
    //
    hwInitializationData.SrbTypeFlags = SRB_TYPE_FLAG_STORAGE_REQUEST_BLOCK;

    // call StorPort to register HW init data
    status = StorPortInitialize(Argument1,
                                Argument2,
                                &hwInitializationData,
                                NULL);

    return status;

} // end DriverEntry()

VOID
GetInterruptMode (
_In_ PAHCI_ADAPTER_EXTENSION    AdapterExtension
)
/*++
    This function determines the interrupt/synchronization model that StorAHCI uses.
    StorAHCI supports two interrupt/synchronization model: 
    1. More than one port share interrupt (line based, single message MSI or there are ports share same interrupt message using MSI).
       StorAHCI looks into IS register to understand which port triggers the interrupt.
       Synchronization will acquire interrupt spinlock for the whole adapter.
    2. Each port has its own interrupt message in MSI mode.
       StorAHCI uses message ID as port number as interrupt source.
       Synchronization will acquire message spinlock.

Called by:
    AhciHwFindAdapter

Note:
    AdapterExtension->HighestPort should have been assigned value before calling this function.

Affected Variables/Registers:
    AdapterExtension->StateFlags.InterruptMessagePerPort

Return Values:
    None
--*/
{
    ULONG status;
    ULONG messageCount = 0;
    MESSAGE_INTERRUPT_INFORMATION messageInfo = {0};

    //
    // For MSI/MSI-X interrupt, use loop to get MSI message info until the API fails.
    // In dump mode, always use interrupt share mode.
    //
    if (!IsDumpMode(AdapterExtension)) {
        AHCI_Global_HBA_CONTROL ghc = {0};

        ghc.AsUlong = StorPortReadRegisterUlong(AdapterExtension, &AdapterExtension->ABAR_Address->GHC.AsUlong);

        //
        // If GHC.MRSM is set to 1, it means that adapter has reverted to single MSI mode.
        //
        if (ghc.MRSM == 0) {
            for (messageCount = 0;; messageCount++) {
                status = StorPortGetMSIInfo(AdapterExtension, messageCount, &messageInfo);

                if (status != STOR_STATUS_SUCCESS) {
                    break;
                }
            }
        }
    }

    if (messageCount > AdapterExtension->HighestPort) {
        AdapterExtension->StateFlags.InterruptMessagePerPort = 1;
    } else {
        AdapterExtension->StateFlags.InterruptMessagePerPort = 0;
    }

    StorPortDebugPrint(3, "StorAHCI - Interrupt Mode: Adapter 0x%04X-0x%04X --- %s\n",
                           AdapterExtension->VendorID,
                           AdapterExtension->DeviceID,
                           (AdapterExtension->StateFlags.InterruptMessagePerPort == 1) ? "MessagePerPort" : "Legacy");

    return;
}


BOOLEAN
AllocateResourcesForAdapter(
    _In_ PAHCI_ADAPTER_EXTENSION         AdapterExtension,
    _In_ PPORT_CONFIGURATION_INFORMATION ConfigInfo,
    _In_range_(1, AHCI_MAX_PORT_COUNT) ULONG PortCount
    )
/*
    Internal function to allocate required memory for Ports

    In AHCI, the controller structures are both the command header list and the received FIS buffer.
        The mechanism for requesting all of this memory is StorPortGetUncachedExtension. This function returns page-aligned address.
        NOTE! StorPortGetUncachedExtension can only be called while in FindAdapter routine.
        NOTE! In order to perform crashdump/hibernate the UncachedExtensionSize should not be larger than 30K.

    Alignment requirement:
    -   Command List Base Addresses: must be 1K aligned, and the Command list is (sizeof (AHCI_COMMAND_HEADER) * cap.NCS), which is some multiple of 32 bytes in length.
    -   The FIS Base Address: must be 256 aligned, and the FIS Receive buffer is sizeof (AHCI_RECEIVED_FIS), 256 bytes in length.
        The Command Table: must be 128 aligned, and is sizeof(AHCI_COMMAND_TABLE), 1280 bytes in length thanks to some padding in the AHCI_COMMAND_TABLE structure.
                           Command Table is per request, resides in AHCI_SRB_EXTENSION for each request.

        Size of nonCachedExtensionSize for every channel should round up to KiloBytes, so that the pointer for next channel is starting at 1K boundary.

        the Command Header is variable and must be padded so the Received FIS is on a 256 boundary.
        Therefore the number of Command Headers must be 256/32 = 8. Round cap.NCS to the next multiple of 8
*/
{
    ULONG alignment = 0x400; // 1K
    BOOLEAN dumpMode = IsDumpMode(AdapterExtension);
    ULONG paddedNCS = 0;
    ULONG paddedSrbExtensionSize = 0;
    ULONG nonCachedExtensionSize = 0;
    ULONG channelExtensionSize = sizeof(AHCI_CHANNEL_EXTENSION);
    PVOID portsChannelExtension = NULL;
    ULONG executionHistorySize = sizeof(EXECUTION_HISTORY) * MAX_EXECUTION_HISTORY_ENTRY_COUNT;
    PVOID portsExecutionHistory = NULL;
    ULONG_PTR left = 0;
    ULONG i = 0;
    ULONG j = 0;

    NT_ASSERT(PortCount >= 1 && PortCount <= AHCI_MAX_PORT_COUNT);

    // 2.1 allocate nonCachedExtension for Hardware access

    // Allocate Identify Data buffer, CommandList, Receive FIS, SRB Extension for Local SRB, INQUIRY buffer
    // paddedNCS is (CAP.NCS+1) rounds up to the next multiple of 8.
    paddedNCS = ((AdapterExtension->CAP.NCS) / 8 + 1) * 8;

    // SrbExtension needs to align to 128 bytes, pad the size to be multiple of 128 bytes
    paddedSrbExtensionSize = ((sizeof(AHCI_SRB_EXTENSION) - 1) / 128 + 1) * 128;

    // size per Port
    nonCachedExtensionSize = sizeof(AHCI_COMMAND_HEADER) * paddedNCS +      // align to 1K, padded to multiple of 256 bytes. (nocachedextension is page aligned)
                             sizeof(AHCI_RECEIVED_FIS) +                    // align to 256 bytes.
                             paddedSrbExtensionSize * 2 +                   // align to 128 bytes. Local.SrbExtension and SenseSrbExtension
                             sizeof(IDENTIFY_DEVICE_DATA) +                 // 512 bytes
                             ATA_BLOCK_SIZE +                               // ReadLogExtPageData --- 512 bytes
                             INQUIRYDATABUFFERSIZE;                         // Inquiry Data

    // round up to KiloBytes if it's not dump mode. this makes sure that NonCachedExtension for next port can align to 1K.
    if (!dumpMode) {
        nonCachedExtensionSize = ((nonCachedExtensionSize - 1) / alignment + 1) * alignment;

        AdapterExtension->NonCachedExtension = StorPortGetUncachedExtension(AdapterExtension,
                                                                            ConfigInfo,
                                                                            nonCachedExtensionSize * PortCount);
    } else {
        // dump mode, address returned from StorPortGetUncachedExtension() is not guaranteed align with 1K.
        // adding 1K into length so that we can start from 1K alignment safely.
        //       NOTE: StorPortAllocatePool is not supported in dump stack, so we allocate ChannelExtension from UnCachedExtension as work around.
        nonCachedExtensionSize += alignment;

        AdapterExtension->NonCachedExtension = StorPortGetUncachedExtension(AdapterExtension,
                                                                            ConfigInfo,
                                                                            (nonCachedExtensionSize + channelExtensionSize) * PortCount);
    }

    if (AdapterExtension->NonCachedExtension == NULL) {

        // we cannot continue if cannot get nonCachedMemory for Channels.
        NT_ASSERT(FALSE);
        return FALSE;
    }

    AhciZeroMemory((PCHAR)AdapterExtension->NonCachedExtension, nonCachedExtensionSize * PortCount);

    // 2.2 allocate resources for Ports which need AHCI_CHANNEL_EXTENSION for each of them.
    if (!dumpMode) {
        ULONG status = STOR_STATUS_SUCCESS;
        // allocate pool and zero the content
        status = StorPortAllocatePool(AdapterExtension,
                                      channelExtensionSize * PortCount,
                                      AHCI_POOL_TAG,
                                      (PVOID*)&portsChannelExtension);

        if ((status != STOR_STATUS_SUCCESS) || (portsChannelExtension == NULL)) {
           // we cannot continue if cannot get memory for ChannelExtension.
            return FALSE;
        }

        status = StorPortAllocatePool(AdapterExtension,
                                      (ULONG)(executionHistorySize * PortCount),
                                      AHCI_POOL_TAG,
                                      (PVOID*)&portsExecutionHistory);

        if ((status != STOR_STATUS_SUCCESS) || (portsExecutionHistory == NULL)) {
           // Continue if cannot get memory for ExecutionHistory.
           NT_ASSERT(FALSE);
        }
    } else {
        // get channelExtension from UnCachedExtension
        portsChannelExtension = (PCHAR)AdapterExtension->NonCachedExtension + (nonCachedExtensionSize * PortCount);
    }

    AhciZeroMemory((PCHAR)portsChannelExtension, channelExtensionSize * PortCount);

    // assign allocated uncachedExension into ChannelExtensions
    for (i = 0; i <= AdapterExtension->HighestPort; i++) {
        if ( (AdapterExtension->PortImplemented & (1 << i)) != 0 ) {
            PCHAR portsUncachedExtension = (PCHAR)AdapterExtension->NonCachedExtension + nonCachedExtensionSize * j;

            // this port is implemented, allocate and initialize extension for the port
            AdapterExtension->PortExtension[i] = (PAHCI_CHANNEL_EXTENSION)((PCHAR)portsChannelExtension + channelExtensionSize * j);
            AdapterExtension->PortExtension[i]->AdapterExtension = AdapterExtension;
            AdapterExtension->PortExtension[i]->PortNumber = i;
            // set ChannelExtension fields that use NonCachedExtension
            if (!dumpMode) {
                AdapterExtension->PortExtension[i]->CommandList = (PAHCI_COMMAND_HEADER)(portsUncachedExtension);
                if (portsExecutionHistory != NULL) {
                    AdapterExtension->PortExtension[i]->ExecutionHistory = (PEXECUTION_HISTORY)((PCHAR)portsExecutionHistory + executionHistorySize * j);
                }
            } else {
                //get the starting pointer; align the starting location to 1K.
                left = ((ULONG_PTR)portsUncachedExtension) % alignment;

                if (left > 0) {
                    AdapterExtension->PortExtension[i]->CommandList = (PAHCI_COMMAND_HEADER)(portsUncachedExtension + alignment - left);
                } else {
                    AdapterExtension->PortExtension[i]->CommandList = (PAHCI_COMMAND_HEADER)(portsUncachedExtension);
                }
            }

            AdapterExtension->PortExtension[i]->ReceivedFIS = (PAHCI_RECEIVED_FIS)((PCHAR)AdapterExtension->PortExtension[i]->CommandList + sizeof(AHCI_COMMAND_HEADER) * paddedNCS);
            AdapterExtension->PortExtension[i]->Local.SrbExtension = (PAHCI_SRB_EXTENSION)((PCHAR)AdapterExtension->PortExtension[i]->ReceivedFIS + sizeof(AHCI_RECEIVED_FIS));
            AdapterExtension->PortExtension[i]->Sense.SrbExtension = (PAHCI_SRB_EXTENSION)((PCHAR)AdapterExtension->PortExtension[i]->Local.SrbExtension + paddedSrbExtensionSize);
            AdapterExtension->PortExtension[i]->DeviceExtension[0].IdentifyDeviceData = (PIDENTIFY_DEVICE_DATA)((PCHAR)AdapterExtension->PortExtension[i]->Sense.SrbExtension + paddedSrbExtensionSize);
            AdapterExtension->PortExtension[i]->DeviceExtension[0].ReadLogExtPageData = (PUSHORT)((PCHAR)AdapterExtension->PortExtension[i]->DeviceExtension[0].IdentifyDeviceData + sizeof(IDENTIFY_DEVICE_DATA));
            AdapterExtension->PortExtension[i]->DeviceExtension[0].InquiryData = (PUCHAR)((PCHAR)AdapterExtension->PortExtension[i]->DeviceExtension[0].ReadLogExtPageData + ATA_BLOCK_SIZE);
            //
            j++;
        }
    }

    return TRUE;
}


ULONG AhciHwFindAdapter(
    _In_ PVOID AdapterExtension,
    _In_ PVOID HwContext,
    _In_ PVOID BusInformation,
    _In_z_ PCHAR ArgumentString,
    _Inout_ PPORT_CONFIGURATION_INFORMATION ConfigInfo,
    _In_ PBOOLEAN Reserved3
    )
/*++
    This function is called by the Storport driver indirectly when handling an IRP_MJ_PnP, IRP_MN_START_DEVICE.
    The adapter is being started. This function is called at PASSIVE LEVEL.
It assumes:
    Resources for the AHCI controller exist in a single Memory Mapped range.
    No Channels have been initialized or started; no IO is outstanding.
Called by:
    Storport

It performs:
    (overview)
    1. Check the working mode and retrieve adapter vendor/model info.
    2. Initialize ChannelExtension with Identifying memory mapped IO resources, Version and CAP.
    3. Enabling the AHCI interface as per AHCI 1.1 section 10.1.2 (part one)
    4. Initializing the IDE_CONTROLLER_CONFIGURATION structure with data from the AHCI Interface
    (details)
    1.1 Get dump mode
    1.2 Gather Vendor,Device,Revision IDs from PCI
    2.1 Initialize adapterExtension with AHCI abar
    2.2 Initialize adapterExtension with version & cap values
    3.1 Turn on AE, reset the controller if AE is already set
        AHCI 1.1 Section 10.1.2 - 1.
        "Indicate that system software is AHCI aware by setting GHC.AE to '1'."
    3.2 Determine which ports are implemented by the HBA
        AHCI 1.1 Section 10.1.2 - 2.
        "Determine which ports are implemented by the HBA, by reading the PI register. This bitmap value will aid software in determining how many ports are available and which port registers need to be initialized."
    3.3 get biggest port number
    3.4 Initializing the rest of PORT_CONFIGURATION_INFORMATION
    3.5 Register Power Setting Change Notification Guids
    4.1 Turn on IE, pending interrupts will be cleared when port starts
        This has to be done after 3.2 because we need to know the number of channels before we check each PxIS.
        Verify that none of the PxIS registers are loaded, but take no action
        Note: Due to the multi-tiered nature of the AHCI HBA's interrupt architecture, system software must always ensure that the PxIS (clear this first) and IS.IPS (clear this second) registers are cleared to '0' before programming the PxIE and GHC.IE registers. This will prevent any residual bits set in these registers from causing an interrupt to be asserted.
        However, the interrupt handler hasn't been hooked up by StorPort yet, so no interrupts will be handled by software until that happens.
    4.2 Initialize the remaining port configuration settings after getting registry settings.
    4.3 Allocate resources for both DMA use and all Channel/Port extensions.
    4.4 Initialize ports and start them. Dump stack will do this when receiving the INQUIRY command

Affected Variables/Registers:
    AdapterExtension->ABAR_Address
    AdapterExtension->CAP
    AdapterExtension->CAP2
    AdapterExtension->Version

    GHC.AE, GHC.IE, GHC.HR
    IS and all fields in the HBA's register memory space except PxFB/PxFBU/PxCLB/PxCLBU that are not HwInit

Return Values:
    The miniport driver returns TRUE if it successfully execute the whole function.
    Any errors causes the driver to return FALSE and prevents the driver from loading.

Note:
    HwStorFindAdapter must set the MaximumTransferLength and NumberOfPhysicalBreaks fields in the ConfigInfo structure.

    Other than these fields, the PORT_CONFIGURATION_INFORMATION (Storport) structure will always fully specify all adapter resources
    that are required to start the adapter.

--*/
{
    ULONG storStatus = STOR_STATUS_UNSUCCESSFUL;
    PAHCI_ADAPTER_EXTENSION adapterExtension = NULL;
    PAHCI_MEMORY_REGISTERS  abar = NULL;
    //Used to find the number of channels implemented
    ULONG                   i = 0;
    ULONG                   piMask = 0;
    ULONG                   numberOfHighestPort = 0;
    ULONG                   portCount = 0;
    ULONG                   portNumber;
    //Used to enable the AHCI interface
    AHCI_Global_HBA_CONTROL ghc = {0};
    //guids
    GUID                    powerSettingChangeGuids[2] = {0};
    BOOLEAN                 dumpMode = FALSE;
    PAHCI_DUMP_CONTEXT      dumpContext = (PAHCI_DUMP_CONTEXT)ConfigInfo->MiniportDumpData;

    UNREFERENCED_PARAMETER(HwContext);
    UNREFERENCED_PARAMETER(BusInformation);
    UNREFERENCED_PARAMETER(ArgumentString);
    UNREFERENCED_PARAMETER(Reserved3);

    adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    // "StoppedState == 0" indicates it's the normal HwFindAdapter call;
    // "StoppedState == 1" indicates the adapter is restarting.
    if (adapterExtension->StateFlags.StoppedState == 0) {
        // use for debugging, assume no more than 4 AHCI adapters in system
        g_AdapterExtension[g_AdapterExtensionIndex++] = AdapterExtension;
        g_AdapterExtensionIndex %= 4;
    }

    //adapterExtension->AdapterNumber = ConfigInfo->SlotNumber;
    adapterExtension->SystemIoBusNumber = ConfigInfo->SystemIoBusNumber;
    adapterExtension->SlotNumber = ConfigInfo->SlotNumber;

    //1.1 Get dump mode
    adapterExtension->DumpMode = ConfigInfo->DumpMode;
    dumpMode = IsDumpMode(adapterExtension);

    if (dumpMode) {
        if (dumpContext != NULL) {
            // In dump/hibernation mode, need to mark ConfigInfo->MiniportDumpData
            // and any embedded memory buffer(s) in MiniportDumpData
            StorPortMarkDumpMemory(AdapterExtension,
                                   dumpContext,
                                   FIELD_OFFSET(AHCI_DUMP_CONTEXT, Ports) +
                                   (dumpContext->PortCount * sizeof(AHCI_DUMP_PORT_CONTEXT)),
                                   0);
        } else {
            NT_ASSERT(FALSE);
            return SP_RETURN_ERROR;
        }
    }

    //1.2 Gather Vendor,Device,Revision IDs from PCI
    if (!dumpMode) {

        //
        // Enumeration may be via either PCI or ACPI. Bus type is set by storport/miniport.
        // Check if we are enumerating via PCI, and retrieve the address appropriately.
        //
        if (ConfigInfo->AdapterInterfaceType == PCIBus) {
            ULONG pcicfgLen = 0;
            UCHAR pcicfgBuffer[0x30] = {0};

            pcicfgLen = StorPortGetBusData(adapterExtension,
                                           PCIConfiguration,
                                           ConfigInfo->SystemIoBusNumber,
                                           (ULONG)ConfigInfo->SlotNumber,
                                           (PVOID)pcicfgBuffer,
                                           (ULONG)0x30);
            if (pcicfgLen == 0x30) {
                PPCI_COMMON_CONFIG pciConfigData = (PPCI_COMMON_CONFIG)pcicfgBuffer;
                adapterExtension->VendorID = pciConfigData->VendorID;
                adapterExtension->DeviceID = pciConfigData->DeviceID;
                adapterExtension->RevisionID = pciConfigData->RevisionID;
                // on PCI bus, AHCI Base Address is BAR5. Bits 0-3 defined for other usages, not part of address value.
                adapterExtension->AhciBaseAddress = pciConfigData->u.type0.BaseAddresses[5] & (0xFFFFFFF0);
            } else {
                NT_ASSERT(FALSE);
                return SP_RETURN_ERROR;
            }
        }
    } else {
        adapterExtension->VendorID = dumpContext->VendorID;
        adapterExtension->DeviceID = dumpContext->DeviceID;
        adapterExtension->RevisionID = dumpContext->RevisionID;
        adapterExtension->AhciBaseAddress = dumpContext->AhciBaseAddress;

        adapterExtension->RegistryFlags = dumpContext->AdapterRegistryFlags;
        adapterExtension->LogFlags = dumpContext->LogFlags;
    }

    //2.1 Initialize adapterExtension with AHCI abar
    abar = GetABARAddress(adapterExtension, ConfigInfo);

    //2.1.1 If abar is still NULL after all of that, malformed resources.  We aren't going to get very far.
    if (abar == NULL) {
        return SP_RETURN_ERROR;
    } else {
        adapterExtension->ABAR_Address = abar;
    }

    //2.2 Initialize adapterExtension with version & cap values
    adapterExtension->Version.AsUlong = StorPortReadRegisterUlong(adapterExtension, &abar->VS.AsUlong);
    adapterExtension->CAP.AsUlong = StorPortReadRegisterUlong(adapterExtension, &abar->CAP.AsUlong);
    adapterExtension->CAP2.AsUlong = StorPortReadRegisterUlong(adapterExtension, &abar->CAP2.AsUlong);

    //3.1 Turn on AE (AHCI 1.1 Section 10.1.2 - 1)
    ghc.AsUlong = StorPortReadRegisterUlong(adapterExtension, &abar->GHC.AsUlong);
    if (ghc.AE == 1) {
        if (!AhciAdapterReset(adapterExtension)) {
            return SP_RETURN_ERROR;
        }
    } //AE is 0. Either through power up or reset we are now pretty sure the controller is in 5.2.2.1    H:Init
    ghc.AsUlong = 0;
    ghc.AE = 1;
    StorPortWriteRegisterUlong(adapterExtension, &abar->GHC.AsUlong, ghc.AsUlong);

    adapterExtension->IS = &abar->IS;

    //3.2 Determine which ports are implemented by the HBA
    if (!dumpMode) {
        // Get implemented ports from PI register (AHCI 1.1 Section 10.1.2 - 2)
        adapterExtension->PortImplemented = StorPortReadRegisterUlong(adapterExtension, &abar->PI);
    } else {
        // Get implemented ports from the dump context
        adapterExtension->PortImplemented = 0;

        for (i = 0; i < dumpContext->PortCount; i++) {
            adapterExtension->PortImplemented |= 1 << dumpContext->Ports[i].PortNumber;
        }
    }

    // At least one port must be implemented
    if (adapterExtension->PortImplemented == 0) {
        return SP_RETURN_ERROR;
    }

    //3.3 Get biggest port number value and implemented port count
    if (!dumpMode) {
        numberOfHighestPort = AHCI_MAX_PORT_COUNT;
        //Check from highest bit to lowest bit for the first highest bit set
        for (piMask = (ULONG)(1 << (AHCI_MAX_PORT_COUNT - 1)); piMask != 0; piMask = (ULONG)(piMask >> 1)){
            numberOfHighestPort--;
            if ((adapterExtension->PortImplemented & piMask) != 0) {
                break;
            }
        }

        for (i = 0; i <= numberOfHighestPort; i++) {
            if ((adapterExtension->PortImplemented & (1 << i)) != 0) {
                portCount++;
            }
        }
    } else {
        numberOfHighestPort = 0;
        for (i = 0; i < dumpContext->PortCount; i++) {
            if (dumpContext->Ports[i].PortNumber > numberOfHighestPort) {
                numberOfHighestPort = (UCHAR)dumpContext->Ports[i].PortNumber;
            }
        }

        portCount = dumpContext->PortCount;
    }

    NT_ASSERT(numberOfHighestPort < AHCI_MAX_PORT_COUNT);
    adapterExtension->HighestPort = numberOfHighestPort;

    //
    // AHCI specification requires that implemented port count (number of bit set in PI register)
    // is less than or equal to CAP.NP + 1
    //
    // Currently StorAHCI does not utilize CAP.NP. It remains useful to identify platform BIOSes
    // which may violate the specification.
    //
    NT_ASSERT(portCount > 0 && portCount <= (ULONG)(adapterExtension->CAP.NP + 1));

    //3.4 Initializing the rest of PORT_CONFIGURATION_INFORMATION
    ConfigInfo->MaximumTransferLength = AHCI_MAX_TRANSFER_LENGTH_DEFAULT;
    ConfigInfo->NumberOfPhysicalBreaks = 0x21;  // Since "NumberOfPhysicalBreaks" has been used in storage stack as the count of entries, use value of physical breaks plus one.
    ConfigInfo->AlignmentMask = 1;              // ATA devices need WORD alignment
    ConfigInfo->ScatterGather = TRUE;
    ConfigInfo->ResetTargetSupported = TRUE;
    ConfigInfo->NumberOfBuses = (UCHAR)(numberOfHighestPort + 1);
    ConfigInfo->MaximumNumberOfTargets = AHCI_MAX_DEVICE;
    ConfigInfo->MaximumNumberOfLogicalUnits = 1 /*AHCI_MAX_LUN*/;   //NOTE: only supports 1 for now. there is a legacy ATAPI device that may have 2 luns.
    // set driver to run in full duplex mode
    ConfigInfo->SynchronizationModel = StorSynchronizeFullDuplex;
    ConfigInfo->BusResetHoldTime = 0;       // StorAHCI wait RESET to be completed by itself, no need for port driver to wait.
    ConfigInfo->MaxNumberOfIO = portCount * adapterExtension->CAP.NCS;

    ConfigInfo->FeatureSupport |= STOR_ADAPTER_FEATURE_STOP_UNIT_DURING_POWER_DOWN;
    ConfigInfo->FeatureSupport |= STOR_ADAPTER_FEATURE_RICH_TEMPERATURE_THRESHOLD;
    ConfigInfo->FeatureSupport |= STOR_ADAPTER_FEATURE_DEVICE_TELEMETRY;
    ConfigInfo->FeatureSupport |= STOR_ADAPTER_DMA_V3_PREFERRED;

    //
    // Set interrupt related entry points.
    //
    if (ConfigInfo->InterruptMode == Latched) {
        //
        // Both MSI and MSI-X are Latched interrupt mode.
        //
        ConfigInfo->HwMSInterruptRoutine = AhciHwMSIInterrupt;
        ConfigInfo->InterruptSynchronizationMode = InterruptSynchronizePerMessage;

        //
        // Allocate MessageGroupAffinity buffer for later use.
        //
        if (!dumpMode) {
            StorPortAllocatePool(AdapterExtension,
                                sizeof(GROUP_AFFINITY) * (adapterExtension->HighestPort + 1),
                                AHCI_POOL_TAG,
                                (PVOID*)&adapterExtension->MessageGroupAffinity);
        }

    } else {
        ConfigInfo->InterruptSynchronizationMode = InterruptSynchronizeAll;
    }

    //3.5 Register Power Setting Change Notification Guids
    if (!dumpMode) {
        /* 0b2d69d7-a2a1-449c-9680-f91c70521c60 -DIPM/HIPM */
        powerSettingChangeGuids[0].Data1 = 0x0b2d69d7;
        powerSettingChangeGuids[0].Data2 = 0xa2a1;
        powerSettingChangeGuids[0].Data3 = 0x449c;
        powerSettingChangeGuids[0].Data4[0] = 0x96;
        powerSettingChangeGuids[0].Data4[1] = 0x80;
        powerSettingChangeGuids[0].Data4[2] = 0xf9;
        powerSettingChangeGuids[0].Data4[3] = 0x1c;
        powerSettingChangeGuids[0].Data4[4] = 0x70;
        powerSettingChangeGuids[0].Data4[5] = 0x52;
        powerSettingChangeGuids[0].Data4[6] = 0x1c;
        powerSettingChangeGuids[0].Data4[7] = 0x60;

        /* DAB60367-53FE-4fbc-825E-521D069D2456 -Adaptive */
        powerSettingChangeGuids[1].Data1 = 0xDAB60367;
        powerSettingChangeGuids[1].Data2 = 0x53FE;
        powerSettingChangeGuids[1].Data3 = 0x4fbc;
        powerSettingChangeGuids[1].Data4[0] = 0x82;
        powerSettingChangeGuids[1].Data4[1] = 0x5E;
        powerSettingChangeGuids[1].Data4[2] = 0x52;
        powerSettingChangeGuids[1].Data4[3] = 0x1D;
        powerSettingChangeGuids[1].Data4[4] = 0x06;
        powerSettingChangeGuids[1].Data4[5] = 0x9D;
        powerSettingChangeGuids[1].Data4[6] = 0x24;
        powerSettingChangeGuids[1].Data4[7] = 0x56;

        StorPortSetPowerSettingNotificationGuids(AdapterExtension, 2, powerSettingChangeGuids);
    }

    //4.1 Turn on IE, pending interrupts will be cleared when port starts
    adapterExtension->LastInterruptedPort = (ULONG)(-1);
    ghc.IE = 1;
    StorPortWriteRegisterUlong(adapterExtension, &abar->GHC.AsUlong, ghc.AsUlong);

    //4.2.2 Configure DMA 64 bit support settings.
    if (adapterExtension->CAP.S64A) {
#if defined (_ARM64_) 
        ConfigInfo->Dma64BitAddresses = SCSI_DMA64_MINIPORT_FULL64BIT_NO_BOUNDARY_REQ_SUPPORTED;
#else
        ConfigInfo->Dma64BitAddresses = SCSI_DMA64_MINIPORT_SUPPORTED;
#endif

        // increase size of SrbExtension to accommodate 64-bit move commands
        // and 1 extra Scripts instruction (turn off 64-bit mode)
        ConfigInfo->SrbExtensionSize += (ULONG)((ConfigInfo->NumberOfPhysicalBreaks + 2) * 4);
    }

    //4.3 allocate resources, implemented port information needs to be ready before calling the following routine.
    if (adapterExtension->StateFlags.StoppedState == 0) {
        if (!AllocateResourcesForAdapter(adapterExtension, ConfigInfo, portCount)) {
            return SP_RETURN_ERROR;
        }
    }

    //
    // Initialize timers.
    //
    // Notes:
    //
    // 1. If we are in dump mode, do not initialize timers. Interrupt is disabled.
    //
    // 2. This initialization has to be done before trying to getting ports into running state (i.e. by
    //    calling AhciAdapterRunAllPorts() later below in this function). The port start process
    //    utilizes StartPortTimer (Note: In dump mode the port start process will be using polling).
    //
    // 3. Currently WorkerTimer is used only for PartialToSlumber during interrupt servicing (not applicable in dump mode)
    //
    if (!dumpMode) {

        for (i = 0; i <= adapterExtension->HighestPort; i++) {

            if (adapterExtension->PortExtension[i] != NULL) {

                if (adapterExtension->PortExtension[i]->StartPortTimer == NULL) {

                    storStatus = StorPortInitializeTimer(AdapterExtension, &adapterExtension->PortExtension[i]->StartPortTimer);

                    if (storStatus != STOR_STATUS_SUCCESS) {
                        NT_ASSERT(FALSE);
                        return SP_RETURN_ERROR;
                    }
                }

                if (adapterExtension->PortExtension[i]->WorkerTimer == NULL) {

                    storStatus = StorPortInitializeTimer(AdapterExtension, &adapterExtension->PortExtension[i]->WorkerTimer);

                    if (storStatus != STOR_STATUS_SUCCESS) {
                        NT_ASSERT(FALSE);
                        return SP_RETURN_ERROR;
                    }
                }

            }
        }
    }

    // reset "StoppedState". NOTE: This field should not be referenced anymore in this function after following line.
    adapterExtension->StateFlags.StoppedState = 0;

    //4.4 initialize ports and start them.
    //4.4.1 initialize all AHCI ports
    for (i = 0; i <= adapterExtension->HighestPort; i++) {
        if (adapterExtension->PortExtension[i] != NULL) {
            // in case of PortInitialize fails, ChannelExtension->StateFlags.Initialized will remain as 'FALSE'. there will be not attempt to start the Port.
            AhciPortInitialize(adapterExtension->PortExtension[i]);
        }
    }

    if (dumpMode) {

        //
        // In dump mode copy registry flags and telemetry configuration from the dump context. If telemetry extends to more than one
        // device the logic should cover migration of per device settings, initially we are limited to a single boot device
        //

        for (i = 0; i < dumpContext->PortCount; i++) {
            portNumber = dumpContext->Ports[i].PortNumber;

            if (adapterExtension->PortExtension[portNumber] != NULL) {

                adapterExtension->PortExtension[portNumber]->RegistryFlags = dumpContext->Ports[i].PortRegistryFlags;

                //
                // If the table in dump context is empty (not filled) - keep static data intact
                //
                /*if (dumpContext->PublicGPLogTableAddresses[0]) {
                    for(Index=0;
                        Index < sizeof(AhciPublicGPLogTableAddresses) / sizeof(AhciPublicGPLogTableAddresses[0] );
                        Index++) {
                        AhciPublicGPLogTableAddresses[Index] = dumpContext->PublicGPLogTableAddresses[Index];
                    }

                    AhciGPLogPageIntoPrivate = dumpContext->PrivateGPLogPageAddress;
                }*/

                StorPortMoveMemory((PVOID)&adapterExtension->PortExtension[portNumber]->DeviceExtension->HybridInfo,
                                   (PVOID)&dumpContext->Ports[i].HybridInfo,
                                   sizeof(GP_LOG_HYBRID_INFORMATION_HEADER));
            }
        }
    }

    return SP_RETURN_FOUND;
}

BOOLEAN
AhciHwInitialize (
    _In_ PVOID AdapterExtension
    )
{
    ULONG                       status = STOR_STATUS_SUCCESS;
    PERF_CONFIGURATION_DATA     perfConfigData = {0};
    PAHCI_ADAPTER_EXTENSION     adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    //
    // Get adapter current interrupt mode. Interrupt Messages can be accessed after HwFindAdapter.
    //
    GetInterruptMode(adapterExtension);

    StorPortEnablePassiveInitialization(AdapterExtension, AhciHwPassiveInitialize);

    //
    // Query perf optimization information
    //
    perfConfigData.Version = STOR_PERF_VERSION;
    perfConfigData.Size = sizeof(PERF_CONFIGURATION_DATA);
    
    status = StorPortInitializePerfOpts(AdapterExtension, TRUE, &perfConfigData);

    //
    // Turn on DPC Redirection if it's supported.
    //
    if ((status == STOR_STATUS_SUCCESS) &&
        ((perfConfigData.Flags & STOR_PERF_DPC_REDIRECTION) != 0)) {

        AhciZeroMemory((PCHAR)&perfConfigData, sizeof(PERF_CONFIGURATION_DATA));
        perfConfigData.Version = STOR_PERF_VERSION;
        perfConfigData.Size = sizeof(PERF_CONFIGURATION_DATA);

        perfConfigData.Flags = STOR_PERF_DPC_REDIRECTION;

        if (adapterExtension->StateFlags.InterruptMessagePerPort == 1) {
            //
            // Limit interrupt redirection to useful messages only.
            //
            if (adapterExtension->MessageGroupAffinity != NULL) {
                AhciZeroMemory((PCHAR)adapterExtension->MessageGroupAffinity, sizeof(GROUP_AFFINITY)* (adapterExtension->HighestPort + 1));

                perfConfigData.Flags |= (STOR_PERF_INTERRUPT_MESSAGE_RANGES | STOR_PERF_ADV_CONFIG_LOCALITY);

                if (adapterExtension->HighestPort > 0) {
                    perfConfigData.FirstRedirectionMessageNumber = 0;
                    perfConfigData.LastRedirectionMessageNumber = adapterExtension->HighestPort;
                }
                else {
                    perfConfigData.FirstRedirectionMessageNumber = 0;
                    perfConfigData.LastRedirectionMessageNumber = 0;
                }

                perfConfigData.MessageTargets = adapterExtension->MessageGroupAffinity;
            }

        }

        //
        // Support concurrent channel so that ports can have StartIo routine running concurrently.
        //
        perfConfigData.Flags |= STOR_PERF_CONCURRENT_CHANNELS;
        perfConfigData.ConcurrentChannels = NumberOfSetBits(adapterExtension->PortImplemented);

        status = StorPortInitializePerfOpts(AdapterExtension, FALSE, &perfConfigData);

        NT_ASSERT(status == STOR_STATUS_SUCCESS);
    }

    //
    // async process to get all ports into running state
    //
    AhciAdapterRunAllPorts(adapterExtension);

    return TRUE;
}

BOOLEAN
AhciHwPassiveInitialize (
    _In_ PVOID AdapterExtension
    )
{
    ULONG i;
    ULONG status = STOR_STATUS_SUCCESS;
    BOOLEAN enableD3Cold = FALSE;
    BOOLEAN d3ColdSupported = FALSE;
    BOOLEAN reportF1State = FALSE;
    ULONG bufferSize = sizeof(STOR_POFX_DEVICE_V2);

    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    // 1. initialize DPC for IO completion
    for (i = 0; i <= adapterExtension->HighestPort; i++) {
        if (adapterExtension->PortExtension[i] != NULL) {
            StorPortInitializeDpc(AdapterExtension, &adapterExtension->PortExtension[i]->CompletionDpc, AhciPortSrbCompletionDpcRoutine);
            StorPortInitializeDpc(AdapterExtension, &adapterExtension->PortExtension[i]->BusChangeDpc, AhciPortBusChangeDpcRoutine);
        }
    }

    // 2. check if ACPI supports turning off power on link
    AhciAdapterEvaluateDSMMethod(adapterExtension);

    //
    // Get and cache D3Cold support.  
    //
    status = StorPortGetD3ColdSupport(AdapterExtension,
                                        NULL,
                                        &d3ColdSupported);
    if (status == STOR_STATUS_SUCCESS) {
        adapterExtension->StateFlags.D3ColdSupported = d3ColdSupported;
    }
    
    if (reportF1State) {
        bufferSize += STOR_POFX_COMPONENT_IDLE_STATE_SIZE;
    }

    // 3. allocate STOR_POFX_DEVICE data structure for adapter, initialize the structure and register for runtime power management.
    status = StorPortAllocatePool(AdapterExtension,
                                  bufferSize,
                                  AHCI_POOL_TAG,
                                  (PVOID*)&adapterExtension->PoFxDevice);

    if (status != STOR_STATUS_SUCCESS) {
        goto Exit;
    }

    AhciZeroMemory((PCHAR)adapterExtension->PoFxDevice, sizeof(STOR_POFX_DEVICE_V2));

    adapterExtension->PoFxDevice->Version = STOR_POFX_DEVICE_VERSION_V2;
    adapterExtension->PoFxDevice->Size = STOR_POFX_DEVICE_V2_SIZE;
    adapterExtension->PoFxDevice->ComponentCount = 1;

    if (IsD3ColdAllowed(adapterExtension)) {
        adapterExtension->PoFxDevice->Flags = STOR_POFX_DEVICE_FLAG_ENABLE_D3_COLD;
    }

    // Indicate miniport opt-in adapter D3 wake support.
    adapterExtension->PoFxDevice->Flags |= STOR_POFX_DEVICE_FLAG_ADAPTER_D3_WAKE;
    
    // Indicate dump miniport can't bring adapter to active
    adapterExtension->PoFxDevice->Flags |= STOR_POFX_DEVICE_FLAG_NO_DUMP_ACTIVE;

    adapterExtension->PoFxDevice->Components[0].Version = STOR_POFX_COMPONENT_VERSION_V1;
    adapterExtension->PoFxDevice->Components[0].Size = STOR_POFX_COMPONENT_SIZE;
    adapterExtension->PoFxDevice->Components[0].FStateCount = reportF1State ? 2 : 1;
    adapterExtension->PoFxDevice->Components[0].DeepestWakeableFState = reportF1State ? 1 : 0;
    adapterExtension->PoFxDevice->Components[0].Id = STORPORT_POFX_ADAPTER_GUID;

    adapterExtension->PoFxDevice->Components[0].FStates[0].Version = STOR_POFX_COMPONENT_IDLE_STATE_VERSION_V1;
    adapterExtension->PoFxDevice->Components[0].FStates[0].Size = STOR_POFX_COMPONENT_IDLE_STATE_SIZE;
    adapterExtension->PoFxDevice->Components[0].FStates[0].TransitionLatency = 0;
    adapterExtension->PoFxDevice->Components[0].FStates[0].ResidencyRequirement = 0;
    adapterExtension->PoFxDevice->Components[0].FStates[0].NominalPower = STOR_POFX_UNKNOWN_POWER;

    if (reportF1State) {
        adapterExtension->StateFlags.UseAdapterF1InsteadOfD3 = FALSE;
        adapterExtension->PoFxDevice->Components[0].FStates[1].Version = STOR_POFX_COMPONENT_IDLE_STATE_VERSION_V1;
        adapterExtension->PoFxDevice->Components[0].FStates[1].Size = STOR_POFX_COMPONENT_IDLE_STATE_SIZE;
        adapterExtension->PoFxDevice->Components[0].FStates[1].TransitionLatency = 1;
        adapterExtension->PoFxDevice->Components[0].FStates[1].ResidencyRequirement = 0;
        adapterExtension->PoFxDevice->Components[0].FStates[1].NominalPower = STOR_POFX_UNKNOWN_POWER;
    }

    // registry runtime power management for Adapter
    status = StorPortInitializePoFxPower(AdapterExtension, NULL, (PSTOR_POFX_DEVICE)adapterExtension->PoFxDevice, &enableD3Cold);

    if (status != STOR_STATUS_SUCCESS) {
        StorPortFreePool(AdapterExtension, adapterExtension->PoFxDevice);
        adapterExtension->PoFxDevice = NULL;
        adapterExtension->StateFlags.PoFxEnabled = FALSE;
        adapterExtension->StateFlags.PoFxActive = FALSE;

        goto Exit;
    }

    // register success
    adapterExtension->StateFlags.PoFxEnabled = TRUE;
    adapterExtension->StateFlags.PoFxActive = TRUE;

Exit:
    return TRUE;
}

VOID
AhciAdapterPrepareForBusReScan(
    _In_ PAHCI_ADAPTER_EXTENSION AdapterExtension
    )
/*++

Routine Description:

    This function do preparation work before StorAHCI can process any device enumeration commands

Arguments:

    AdapterExtension - Pointer to the device extension for adapter.

Return Value:

    None

--*/
{
    // 1. If a port/device is powered off, power it on.
    if (AdapterExtension->StateFlags.SupportsAcpiDSM == 1) {
        // ACPI method is implemented, invoke ACPI method to power on all ports and devices
        AhciPortAcpiDSMControl(AdapterExtension, (ULONG)-1, FALSE);
    }

    return;
}

__inline
VOID
AdapterStop (
    _In_ PAHCI_ADAPTER_EXTENSION AdapterExtension
)
/*++

Routine Description:

    This function stops all the ports of the adapter.

Arguments:

    AdapterExtension - Pointer to the device extension for adapter.

    Note: StartIo spin lock must be held before this function is invoked.

Return Value:

    None

--*/
{
    ULONG   i;

    for (i = 0; i <= AdapterExtension->HighestPort; i++) {
        if (AdapterExtension->PortExtension[i] != NULL) {
            AhciPortStop(AdapterExtension->PortExtension[i]);
        }
    }

    // clear ABAR
    AdapterExtension->ABAR_Address = 0;

    AdapterExtension->StateFlags.StoppedState = 1;

    AdapterReleaseActiveReference(AdapterExtension);

    // clear this bit indicating the work has been finished
    AdapterExtension->PoFxPendingWork.AdapterStop = 0;
}

VOID
AhciAdapterStop (
    _In_ PAHCI_ADAPTER_EXTENSION AdapterExtension
)
/*
    Adapter is being stopped.
*/
{
    // stop all ports.
    ULONG   adapterStopInProcess;
    BOOLEAN adapterIdle = FALSE;

    if (AdapterExtension->StateFlags.StoppedState == 1) {
        // nothing to do if the adapter is already stopped.
        return;
    }

    adapterStopInProcess = InterlockedBitTestAndSet((LONG*)&AdapterExtension->PoFxPendingWork, 0);  //AdapterStop is at bit 0

    if (adapterStopInProcess == 1) {
        // adapter Stop is pending in another process.
        return;
    }

    AdapterAcquireActiveReference(AdapterExtension, &adapterIdle);

    if (!adapterIdle) {
        ULONG   adapterStopPending;
        adapterStopPending = InterlockedBitTestAndReset((LONG*)&AdapterExtension->PoFxPendingWork, 0);  //AdapterStop is at bit 0

        // adapter was in active state, perform AdapterStop.
        // if adapter was in idle state, this work will be done in PoFxActive callback
        if (adapterStopPending == 1) {
            // Reference will be released in AdapterStop() function
            AdapterStop(AdapterExtension);
        }
    }

    return;
}

VOID
AhciAdapterRemoval (
    _In_ PAHCI_ADAPTER_EXTENSION AdapterExtension
)
/*
    Release resources allocated for the adapter and its managed ports/devices
*/
{
    ULONG i;
    PVOID bufferToFree = NULL;
    PVOID executionHistoryToFree = NULL;

    if (IsDumpMode(AdapterExtension)) {
        return;
    }

    if (AdapterExtension->PoFxDevice != NULL) {
        StorPortFreePool(AdapterExtension, AdapterExtension->PoFxDevice);
        AdapterExtension->PoFxDevice = NULL;
        AdapterExtension->StateFlags.PoFxEnabled = FALSE;
        AdapterExtension->StateFlags.PoFxActive = FALSE;
    }

    if (AdapterExtension->MessageGroupAffinity != NULL) {
        StorPortFreePool(AdapterExtension, AdapterExtension->MessageGroupAffinity);
        AdapterExtension->MessageGroupAffinity = NULL;
    }

    // release portsChannelExtension. the whole buffer is sliced for Ports, only need to free the first ChannelExtension.
    for (i = 0; i <= AdapterExtension->HighestPort; i++) {
        if (AdapterExtension->PortExtension[i] != NULL) {
            if (bufferToFree == NULL) {
                bufferToFree = AdapterExtension->PortExtension[i];
            }

            if (AdapterExtension->PortExtension[i]->StartPortTimer != NULL) {
                StorPortFreeTimer(AdapterExtension, AdapterExtension->PortExtension[i]->StartPortTimer);
                AdapterExtension->PortExtension[i]->StartPortTimer = NULL;
            }
            if (AdapterExtension->PortExtension[i]->WorkerTimer != NULL) {
                StorPortFreeTimer(AdapterExtension, AdapterExtension->PortExtension[i]->WorkerTimer);
                AdapterExtension->PortExtension[i]->WorkerTimer = NULL;
            }
            if (AdapterExtension->PortExtension[i]->BusChangeTimer != NULL) {
                StorPortFreeTimer(AdapterExtension, AdapterExtension->PortExtension[i]->BusChangeTimer);
                AdapterExtension->PortExtension[i]->BusChangeTimer = NULL;
            }

            if (AdapterExtension->PortExtension[i]->DeviceInitCommands.CommandTaskFile != NULL) {
                StorPortFreePool(AdapterExtension, (PVOID)AdapterExtension->PortExtension[i]->DeviceInitCommands.CommandTaskFile);
                AdapterExtension->PortExtension[i]->DeviceInitCommands.CommandTaskFile = NULL;
            }

            if (AdapterExtension->PortExtension[i]->PoFxDevice != NULL) {
                StorPortFreePool(AdapterExtension, AdapterExtension->PortExtension[i]->PoFxDevice);
                AdapterExtension->PortExtension[i]->PoFxDevice = NULL;
                AdapterExtension->PortExtension[i]->StateFlags.PoFxEnabled = FALSE;
                AdapterExtension->PortExtension[i]->StateFlags.PoFxActive = FALSE;
            }

            if ((executionHistoryToFree == NULL) && (AdapterExtension->PortExtension[i]->ExecutionHistory != NULL)) {
                executionHistoryToFree = AdapterExtension->PortExtension[i]->ExecutionHistory;
            }

            if (AdapterExtension->PortExtension[i]->ExecutionHistory != NULL) {
                AdapterExtension->PortExtension[i]->ExecutionHistory = NULL;
            }

            AdapterExtension->PortExtension[i] = NULL;
        }
    }

    if (bufferToFree != NULL) {
        StorPortFreePool(AdapterExtension, bufferToFree);
    }

    if (executionHistoryToFree != NULL) {
        StorPortFreePool(AdapterExtension, executionHistoryToFree);
    }

    return;
}


SCSI_ADAPTER_CONTROL_STATUS
AhciHwAdapterControl (
    _In_ PVOID AdapterExtension,
    _In_ SCSI_ADAPTER_CONTROL_TYPE ControlType,
    _In_ PVOID Parameters
    )
/*++

Routine Description:

    This is a generic routine to allow for special adapter control activities.
    It mainly handles power change notifications for the adapter.

Arguments:

    AdapterExtension - Pointer to the device extension for adapter.

    ControlType - Specifies the type of call being made through this routine.

    Parameters - Pointer to parameters needed for this control type (optional).

Return Value:

    SCSI_ADAPTER_CONTROL_STATUS - currently either:
        ScsiAdapterControlSuccess (= 0)
        ScsiAdapterControlUnsuccessful (= 1)

--*/

{
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;
    PSCSI_SUPPORTED_CONTROL_TYPE_LIST controlTypeList;
    SCSI_ADAPTER_CONTROL_STATUS status = ScsiAdapterControlSuccess;

    switch (ControlType) {
        // determine which control types (routines) are supported
        case ScsiQuerySupportedControlTypes:
            // get pointer to control type list
            controlTypeList = (PSCSI_SUPPORTED_CONTROL_TYPE_LIST)Parameters;

            // Report StopAdapter and RestartAdapter are supported.
            if (ScsiQuerySupportedControlTypes < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiQuerySupportedControlTypes] = TRUE;
            }
            if (ScsiStopAdapter < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiStopAdapter] = TRUE;
            }
            if (ScsiRestartAdapter < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiRestartAdapter] = TRUE;
            }
            if (ScsiPowerSettingNotification < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiPowerSettingNotification] = TRUE;
            }
            if (ScsiAdapterPoFxPowerActive < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiAdapterPoFxPowerActive] = TRUE;
            }
            if (ScsiAdapterPoFxPowerSetFState < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiAdapterPoFxPowerSetFState] = TRUE;
            }
            if (ScsiAdapterPower < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiAdapterPower] = TRUE;
            }
            if (ScsiAdapterPrepareForBusReScan < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiAdapterPrepareForBusReScan] = TRUE;
            }
            if (ScsiAdapterSystemPowerHints < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiAdapterSystemPowerHints] = TRUE;
            }
            if (ScsiAdapterSurpriseRemoval < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiAdapterSurpriseRemoval] = TRUE;
            }

            break;

        // ScsiStopAdapter maybe called with PnP or Power activities.
        // StorAHCI supports PnP Srb and ScsiAdapterPower thus does nothing for ScsiStopAdapter.
        // NOTE: ScsiStopAdapter is called after PnP call -StorRemoveDevice or StorSurpriseRemoval,
        // thus resource release should be done here.
        case ScsiStopAdapter:
            if (adapterExtension->StateFlags.Removed == 1) {
                AhciAdapterRemoval(adapterExtension);
            }
            break;

        // ScsiRestartAdapter is called during adapter powering up.
        // StorAHCI supports PnP Srb and ScsiAdapterPower thus does nothing for ScsiRestartAdapter. return value is ScsiAdapterControlSuccess.
        case ScsiRestartAdapter:
            break;

        // ScsiPowerSettingNotification is called when Storport receives notification from Power Manager when power setting is changed.
        case ScsiPowerSettingNotification:
            AhciAdapterPowerSettingNotification(adapterExtension, (PSTOR_POWER_SETTING_INFO)Parameters);
            break;

        case ScsiAdapterPoFxPowerActive: {
            PSTOR_POFX_ACTIVE_CONTEXT activeContext = (PSTOR_POFX_ACTIVE_CONTEXT)Parameters;

            if (AdapterPoFxEnabled(adapterExtension)) {
                adapterExtension->StateFlags.PoFxActive = activeContext->Active ? 1 : 0;

                if (activeContext->Active) {
                    ULONG   adapterStopPending;
                    adapterStopPending = InterlockedBitTestAndReset((LONG*)&adapterExtension->PoFxPendingWork, 0);  //AdapterStop is at bit 0

                    if (adapterStopPending == 1) {
                        //perform pending Adapter Stop work
                        STOR_LOCK_HANDLE lockhandle = {StartIoLock, 0};

                        NT_ASSERT(adapterExtension->StateFlags.StoppedState == 0);

                        // Storport does not acquire any spinlock before calling 
                        // miniport's HwAdapterControl routine. Need to acquire
                        // StartIo spin lock before calling AdapterStop as this is 
                        // required by AdapterStop.
                        StorPortAcquireSpinLock(AdapterExtension, StartIoLock, NULL, &lockhandle);
                        AdapterStop(adapterExtension);
                        StorPortReleaseSpinLock(AdapterExtension, &lockhandle);
                    }
                }
            }
            break;
        }

        case ScsiAdapterPoFxPowerSetFState: {
            break;
        }

        case ScsiAdapterPower: {
            PSTOR_ADAPTER_CONTROL_POWER adapterPower = (PSTOR_ADAPTER_CONTROL_POWER)Parameters;

            NT_ASSERT(adapterPower != NULL);

            StorPortDebugPrint(3, "StorAHCI - LPM: Adapter SystemIoBusNumber:%d - %s\n", 
                adapterExtension->SystemIoBusNumber, adapterPower->PowerState == StorPowerDeviceD0 ? "D0" : "D3");

            if (adapterPower->PowerState == StorPowerDeviceD0) {
                AhciAdapterPowerUp(adapterExtension);                  //power up
            } else if (adapterPower->PowerState == StorPowerDeviceD3) {
                AhciAdapterPowerDown(adapterExtension);                //power down
            } else {
                // AHCI adapter does not support D1 or D2, assert but do not fail the call
                NT_ASSERT(FALSE);
            }
            break;
        }

        case ScsiAdapterPrepareForBusReScan:
            AhciAdapterPrepareForBusReScan(adapterExtension);
            break;

        case ScsiAdapterSystemPowerHints: {
            PSTOR_SYSTEM_POWER_HINTS powerHints = (PSTOR_SYSTEM_POWER_HINTS)Parameters;            

            if (powerHints->Size >= sizeof(STOR_SYSTEM_POWER_HINTS)) {

                StorPortDebugPrint(3, "StorAHCI - LPM: Adapter SystemIoBusNumber:%d - System Power Hint - State: %u - Latency: %u ms\n", 
                    adapterExtension->SystemIoBusNumber, powerHints->SystemPower, powerHints->ResumeLatencyMSec);

                adapterExtension->SystemPowerHintState = powerHints->SystemPower;
                adapterExtension->SystemPowerResumeLatencyMSec = powerHints->ResumeLatencyMSec;

                if (adapterExtension->TracingEnabled) {
                    StorPortEtwEvent2(AdapterExtension,
                                      NULL,
                                      AhciEtwEventSystemPowerHint,
                                      L"Sys Power Hint",
                                      STORPORT_ETW_EVENT_KEYWORD_POWER,
                                      StorportEtwLevelInformational,
                                      StorportEtwEventOpcodeInfo,
                                      NULL,
                                      L"State",
                                      powerHints->SystemPower,
                                      L"Latency (ms)",
                                      powerHints->ResumeLatencyMSec);
                }

            } else {
                status = ScsiAdapterControlUnsuccessful;
            }
            break;
        }

        case ScsiAdapterSurpriseRemoval: {
            //
            // For AHCI, all the cleanup that needs to happen for adapter surprise removals is done in
            // the several ScsiUnitSurpriseRemovals that will be sent down for each LUN. There is no
            // extra work needed to handle adapter surprise removals.
            //
            status = ScsiAdapterControlSuccess;
            break;
        }

        default:
            status = ScsiAdapterControlUnsuccessful;
            break;

    } // end of switch

    return status;

} // AhciHwAdapterControl

BOOLEAN
AhciHwResetBus (
    _In_ PVOID AdapterExtension,
    _In_ ULONG PathId
    )
/*
    Used to Reset the PathId/Port/Channel
*/
{
    BOOLEAN status = FALSE;
    STOR_LOCK_HANDLE lockhandle = {InterruptLock, 0};
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    if ( IsPortValid(adapterExtension, PathId) ) {
        AhciInterruptSpinlockAcquire(adapterExtension, MAXULONG, &lockhandle);
        status = AhciPortReset(adapterExtension->PortExtension[PathId], TRUE);
        AhciInterruptSpinlockRelease(adapterExtension, MAXULONG, &lockhandle);
        adapterExtension->PortExtension[PathId]->DeviceExtension[0].IoRecord.PortDriverResetCount++;

        AhciTelemetryLogResetErrorRecovery(adapterExtension->PortExtension[PathId],
                                          (PSTOR_ADDRESS)&(adapterExtension->PortExtension[PathId]->DeviceExtension->DeviceAddress),
                                          AhciTelemetryEventIdResetBus,
                                          "AhciHwResetBus",
                                          0,
                                          "PortResetStatus",
                                          status
                                          );
    }

    return status;
}

BOOLEAN
AhciHwBuildIo (
    _In_ PVOID AdapterExtension,
    _In_ PSCSI_REQUEST_BLOCK Srb
    )
{
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;
    PAHCI_CHANNEL_EXTENSION channelExtension = NULL;
    UCHAR                   pathId = SrbGetPathId(Srb);
    ULONG                   function = SrbGetSrbFunction(Srb);
    ULONG                   srbFlags = SrbGetSrbFlags(Srb);
    PAHCI_SRB_EXTENSION     srbExtension = GetSrbExtension((PSTORAGE_REQUEST_BLOCK)Srb);

    //
    // Make sure the incoming Srb with the expected type
    //
    NT_ASSERT(Srb->Function == SRB_FUNCTION_STORAGE_REQUEST_BLOCK);

    //
    // SrbStatus value should have been set to pending.
    //
    NT_ASSERT(Srb->SrbStatus == SRB_STATUS_PENDING);

    // SrbExtension is not Null-ed by Storport, so do it here.
    // Zero PRDT later when elements number in the SGL is available.
    AhciZeroMemory((PCHAR)(&(srbExtension->AtaFunction)), (sizeof(AHCI_SRB_EXTENSION) - FIELD_OFFSET(AHCI_SRB_EXTENSION, AtaFunction)));
    AhciZeroMemory((PCHAR)srbExtension, FIELD_OFFSET(AHCI_COMMAND_TABLE, PRDT));

    channelExtension = adapterExtension->PortExtension[pathId];

    if ( IsPortValid(adapterExtension, pathId) &&
         (function == SRB_FUNCTION_EXECUTE_SCSI) &&
         (channelExtension->StateFlags.PowerDown == TRUE)) {
        AhciPortPowerUp(channelExtension);
    }

    //
    // Prepare the Device Command before StartIo routine for SRB_FUNCTION_IO_CONTROL and SRB_FUNCTION_EXECUTE_SCSI requests.
    // This can reduce the lock contention between StartIo and Interrupt locks.
    //
    if ((function == SRB_FUNCTION_EXECUTE_SCSI) ||
        ((function == SRB_FUNCTION_IO_CONTROL) && ((srbFlags & SRB_IOCTL_FLAGS_ADAPTER_REQUEST) == 0))) {

        if ( !IsPortValid(adapterExtension, pathId) ) {
            Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
            StorPortEtwChannelEvent2(adapterExtension,
                                     NULL,
                                     StorportEtwEventOperational,
                                     AhciEtwEventBuildIO,
                                     L"No device",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     StorportEtwLevelError,
                                     StorportEtwEventOpcodeInfo,
                                     Srb,
                                     L"function",
                                     function,
                                     L"srbFlags",
                                     srbFlags);
            goto exit;
        }

        //
        // If the adapter was removed, fail this request.
        // We first check to see if the adapter is removable so that we can
        // avoid incurring a register access if it is not removable.
        //
        if (IsAdapterRemovable(adapterExtension) &&
            IsAdapterRemoved(adapterExtension)) {
            Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
            StorPortEtwChannelEvent2(adapterExtension,
                                     (channelExtension != NULL) ? ((PSTOR_ADDRESS) &(channelExtension->DeviceExtension->DeviceAddress)) : (NULL),
                                     StorportEtwEventOperational,
                                     AhciEtwEventBuildIO,
                                     L"Adapter removed during BuildIo",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     StorportEtwLevelError,
                                     StorportEtwEventOpcodeInfo,
                                     Srb,
                                     L"function",
                                     function,
                                     L"srbFlags",
                                     srbFlags);
            goto exit;
        }
    }

    switch (function) {
        //
        // Get the flag from Storport that tells us if the adapter is removable or not.
        //
        case SRB_FUNCTION_PNP: {
            PSRBEX_DATA_PNP pnpData = (PSRBEX_DATA_PNP)SrbGetSrbExDataByType((PSTORAGE_REQUEST_BLOCK)Srb, SrbExDataTypePnP);
            if (((pnpData->SrbPnPFlags & SRB_PNP_FLAGS_ADAPTER_REQUEST)) &&
                (pnpData->PnPAction == StorQueryCapabilities) &&
                (SrbGetDataTransferLength(Srb) >= sizeof(STOR_DEVICE_CAPABILITIES_EX))) {
                PSTOR_DEVICE_CAPABILITIES_EX storCapabilities = (PSTOR_DEVICE_CAPABILITIES_EX)SrbGetDataBuffer(Srb);
                adapterExtension->StateFlags.Removable = storCapabilities->Removable;
                Srb->SrbStatus = SRB_STATUS_SUCCESS;
            }
            break;
        }

        case SRB_FUNCTION_IO_CONTROL: {

            if ((srbFlags & SRB_IOCTL_FLAGS_ADAPTER_REQUEST) == 0) {

                IOCTLtoATA(adapterExtension->PortExtension[pathId], (PSTORAGE_REQUEST_BLOCK)Srb);

                if (srbExtension->AtaFunction != 0) {
                    if ( ( srbExtension->Sgl == NULL ) && ( IsDataTransferNeeded((PSTORAGE_REQUEST_BLOCK)Srb) )  ) {
                        srbExtension->Sgl = (PLOCAL_SCATTER_GATHER_LIST)StorPortGetScatterGatherList(adapterExtension, Srb);
                    }
                    if (srbExtension->Sgl != NULL) {
                        // Zero PRDT according to SGL elements number to avoid unnecessary CPU usage.
                        AhciZeroMemory((PCHAR)(srbExtension->CommandTable.PRDT), (srbExtension->Sgl->NumberOfElements * sizeof(AHCI_PRDT)));
                    }
                }
            }

            break;
        }

        case SRB_FUNCTION_EXECUTE_SCSI: {

            SCSItoATA(adapterExtension->PortExtension[pathId], (PSTORAGE_REQUEST_BLOCK)Srb);

            if (srbExtension->AtaFunction != 0) {
                if ( ( srbExtension->Sgl == NULL ) && ( IsDataTransferNeeded((PSTORAGE_REQUEST_BLOCK)Srb) )  ) {
                    srbExtension->Sgl = (PLOCAL_SCATTER_GATHER_LIST)StorPortGetScatterGatherList(adapterExtension, Srb);
                }
                if (srbExtension->Sgl != NULL) {
                    // Zero PRDT according to SGL elements number to avoid unnecessary CPU usage.
                    AhciZeroMemory((PCHAR)(srbExtension->CommandTable.PRDT), (srbExtension->Sgl->NumberOfElements * sizeof(AHCI_PRDT)));
                }
            }

            break;
        }

        default: {
            //
            // All other requests will be processed in StartIo routine.
            //
            break;
        }
    }

exit:

    if (Srb->SrbStatus != SRB_STATUS_PENDING) {
        //
        // This routine should return FALSE when SRB is completed.
        //
        StorPortNotification(RequestComplete, AdapterExtension, Srb);

        return FALSE;
    } else {
        //
        // This routine should return TRUE when SRB is not completed.
        //
        return TRUE;
    }
}

BOOLEAN
AhciHwStartIo (
    _In_ PVOID AdapterExtension,
    _In_ PSCSI_REQUEST_BLOCK Srb
    )
/*
    1. Process Adapter request
    2. Bail out if it's adapter request
    3. Validate Port Number, if not valid, bail out.
    4. Process Device/Port request

*/
{
    STOR_LOCK_HANDLE lockhandle = {InterruptLock, 0};
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;
    ULONG function = SrbGetSrbFunction(Srb);
    UCHAR pathId = SrbGetPathId(Srb);
    BOOLEAN adapterRequest = FALSE;
    BOOLEAN processIO = FALSE;

    //1 Work on Adapter requests
    switch (function) {
        case SRB_FUNCTION_PNP: {
            PSRBEX_DATA_PNP pnpData = (PSRBEX_DATA_PNP)SrbGetSrbExDataByType((PSTORAGE_REQUEST_BLOCK)Srb, SrbExDataTypePnP);

            NT_ASSERT(pnpData != NULL);

            if ( (pnpData->SrbPnPFlags & SRB_PNP_FLAGS_ADAPTER_REQUEST ) != 0 ) {
                // only process PnP request for adapter here.
                adapterRequest = TRUE;

                if ( (pnpData->PnPAction == StorRemoveDevice) || (pnpData->PnPAction == StorSurpriseRemoval) ) {
                    // the adapter is going to be removed, mark the state and release all resources allocated later in AdapterControl - ScsiStopAdapter.
                    adapterExtension->StateFlags.Removed = 1;
                    Srb->SrbStatus = SRB_STATUS_SUCCESS;
                } else if (pnpData->PnPAction == StorStopDevice) {
                    AhciAdapterStop(adapterExtension);
                    Srb->SrbStatus = SRB_STATUS_SUCCESS;
                } else {
                    Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
                }

                StorPortEtwChannelEvent2(adapterExtension,
                                         NULL,
                                         (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwEventDiagnostic : StorportEtwEventOperational,
                                         AhciEtwEventStartIO,
                                         L"Processing PNP",
                                         STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                         (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwLevelInformational : StorportEtwLevelError,
                                         StorportEtwEventOpcodeInfo,
                                         Srb,
                                         L"PnPAction",
                                         pnpData->PnPAction,
                                         L"SrbStatus",
                                         Srb->SrbStatus);

                //complete all Adapter PnP request
                StorPortNotification(RequestComplete, AdapterExtension, Srb);
            }
            break;
        }
        case SRB_FUNCTION_POWER: {
            PSRBEX_DATA_POWER powerData = (PSRBEX_DATA_POWER)SrbGetSrbExDataByType((PSTORAGE_REQUEST_BLOCK)Srb, SrbExDataTypePower);

            NT_ASSERT(powerData != NULL);

            if ((powerData->SrbPowerFlags & SRB_POWER_FLAGS_ADAPTER_REQUEST) != 0) {
                // only process Power request for adapter here.
                adapterRequest = TRUE;

                // StorAHCI supports ScsiAdapterPower, thus Storport should not send adapter power Srb to this miniport.
                // Complete the request as SUCCESS anyway.
                NT_ASSERT(FALSE);
                Srb->SrbStatus = SRB_STATUS_SUCCESS;
                StorPortNotification(RequestComplete, AdapterExtension, Srb);
            }
            break;
        }

        case SRB_FUNCTION_FLUSH:  {
            adapterRequest = TRUE;
            //FLUSH Adapter cache. No cache, complete the request.
            Srb->SrbStatus = SRB_STATUS_SUCCESS;
            StorPortNotification(RequestComplete, AdapterExtension, Srb);
            break;
        }

        case SRB_FUNCTION_IO_CONTROL: {
            ULONG   srbFlags = SrbGetSrbFlags(Srb);

            if ((srbFlags & SRB_IOCTL_FLAGS_ADAPTER_REQUEST) != 0) {
                adapterRequest = AdapterProcessIOCTL(adapterExtension, (PSTORAGE_REQUEST_BLOCK)Srb);

                if (adapterRequest) {
                    StorPortNotification(RequestComplete, AdapterExtension, Srb);
                }
            }

            break;
        }
    }

    // 1.1 if the request is for adapter, it has been processed. Return from here.
    if (adapterRequest) {
        return TRUE;
    }

    // 2.1 All requests reach here should be for devices
    if ( !IsPortValid(adapterExtension, pathId) ) {
        Srb->SrbStatus = SRB_STATUS_NO_DEVICE;

        StorPortEtwChannelEvent8(adapterExtension,
                                 NULL,
                                 StorportEtwEventOperational,
                                 AhciEtwEventStartIO,
                                 L"Port has no device",
                                 STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                 StorportEtwLevelError,
                                 StorportEtwEventOpcodeInfo,
                                 Srb,
                                 L"AdapterNumber",
                                 adapterExtension->AdapterNumber,
                                 L"SystemIoBusNumber",
                                 adapterExtension->SystemIoBusNumber,
                                 L"SlotNumber",
                                 adapterExtension->SlotNumber,
                                 L"AhciBaseAddress",
                                 adapterExtension->AhciBaseAddress,
                                 L"PathId",
                                 pathId,
                                 L"Function",
                                 Srb->Function,
                                 L"SrbStatus",
                                 Srb->SrbStatus,
                                 NULL,
                                 0);

        StorPortNotification(RequestComplete, AdapterExtension, Srb);
        return TRUE;
    }

    // 2.2 work on device requests
    // Get channelExtension and storAddress to use for ETW Event logging
    PAHCI_CHANNEL_EXTENSION channelExtension = adapterExtension->PortExtension[pathId];
    PSTOR_ADDRESS storAddress = (PSTOR_ADDRESS)&(channelExtension->DeviceExtension->DeviceAddress);

    switch (function) {
        case SRB_FUNCTION_RESET_BUS:    // this one may come from class driver, not port driver. same as AhciHwResetBus
        case SRB_FUNCTION_RESET_DEVICE:
        case SRB_FUNCTION_RESET_LOGICAL_UNIT: {
            
            // these reset requests target to Port
            AhciInterruptSpinlockAcquire(adapterExtension, pathId, &lockhandle);
            Srb->SrbStatus = AhciPortReset(adapterExtension->PortExtension[pathId], TRUE) ? SRB_STATUS_SUCCESS : SRB_STATUS_ERROR;
            StorPortNotification(RequestComplete, AdapterExtension, Srb);
            AhciInterruptSpinlockRelease(adapterExtension, pathId, &lockhandle);
            adapterExtension->PortExtension[pathId]->DeviceExtension[0].IoRecord.PortDriverResetCount++;

            AhciTelemetryLogResetErrorRecovery(adapterExtension->PortExtension[pathId],
                                              (PSTOR_ADDRESS)&(adapterExtension->PortExtension[pathId]->DeviceExtension->DeviceAddress),
                                              AhciTelemetryEventIdResetDeviceRequest,
                                              "AhciHwStartIo RESET Request",
                                              0,
                                              "PortResetStatus",
                                              Srb->SrbStatus
                                              );
            break;
        }

        case SRB_FUNCTION_PNP: {
            PSRBEX_DATA_PNP pnpData = (PSRBEX_DATA_PNP)SrbGetSrbExDataByType((PSTORAGE_REQUEST_BLOCK)Srb, SrbExDataTypePnP);

            NT_ASSERT(pnpData != NULL);

            if ( ((pnpData->SrbPnPFlags & SRB_PNP_FLAGS_ADAPTER_REQUEST) == 0) &&
                 (pnpData->PnPAction == StorQueryCapabilities) &&
                 (SrbGetDataTransferLength(Srb) >= sizeof(STOR_DEVICE_CAPABILITIES_EX)) ) {
                // process StorQueryCapabilities request for device, not adapter.
                // fill in fields of STOR_DEVICE_CAPABILITIES_EX
                PSTOR_DEVICE_CAPABILITIES_EX storCapabilities = (PSTOR_DEVICE_CAPABILITIES_EX)SrbGetDataBuffer(Srb);
                ULONG portProperties = adapterExtension->PortExtension[pathId]->PortProperties;

                storCapabilities->Removable = (portProperties & PORT_PROPERTIES_EXTERNAL_PORT) ? 1 : 0;
                storCapabilities->DeviceD1 = 0;
                storCapabilities->DeviceD2 = 0;
                storCapabilities->LockSupported = 0;
                storCapabilities->EjectSupported = 0;
                storCapabilities->DockDevice = 0;
                storCapabilities->UniqueID = 0;         //no uniqueID, let PnP generates one
                storCapabilities->SilentInstall = 1;
                storCapabilities->SurpriseRemovalOK = 0;
                storCapabilities->NoDisplayInUI = 0;

                Srb->SrbStatus = SRB_STATUS_SUCCESS;
                StorPortNotification(RequestComplete, AdapterExtension, Srb);

            } else {
                Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
                StorPortNotification(RequestComplete, AdapterExtension, Srb);
            }

            StorPortEtwChannelEvent2(adapterExtension,
                                     storAddress,
                                     (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwEventDiagnostic : StorportEtwEventOperational,
                                     AhciEtwEventStartIO,
                                     L"Completed PnP request",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwLevelInformational : StorportEtwLevelError,
                                     StorportEtwEventOpcodeInfo,
                                     Srb,
                                     L"SrbPnPFlags",
                                     pnpData->SrbPnPFlags,
                                     L"PnPAction",
                                     pnpData->PnPAction);

            break;
        }

        case SRB_FUNCTION_SHUTDOWN: {
            //This request is for the device to flush device data from adapter cache.
            //The miniport driver must hold on to the shutdown request until no data remains in the HBA's internal cache for the target logical unit and,
            //then, complete the shutdown request.
            PATA_DEVICE_PARAMETERS  deviceParameters = &adapterExtension->PortExtension[pathId]->DeviceExtension->DeviceParameters;
            BOOLEAN                 sendStandby = (IsDumpHiberMode(adapterExtension) || IsDumpCrashMode(adapterExtension));

            deviceParameters->StateFlags.SystemPoweringDown = TRUE;

            USHORT *eventDescription;
            if (adapterExtension->PortExtension[pathId]->DeviceExtension->SupportedCommands.SetDateAndTime == 0x1) {
                IssueSetDateAndTimeCommand(adapterExtension->PortExtension[pathId], Srb, sendStandby);
                processIO = TRUE;
                eventDescription = L"Shutdown set date and time";
            } else if (sendStandby) {
                //in dump mode, this is the last Srb sent after SYNC CACHE, spin down the disk
                PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension((PSTORAGE_REQUEST_BLOCK)Srb);
                srbExtension->AtaFunction = ATA_FUNCTION_ATA_COMMAND;
                SetCommandReg((&srbExtension->TaskFile.Current), IDE_COMMAND_STANDBY_IMMEDIATE);
                processIO = TRUE;
                eventDescription = L"Shutdown standby immediate";
            } else {
                Srb->SrbStatus = SRB_STATUS_SUCCESS;
                eventDescription = L"Shutdown";
                StorPortNotification(RequestComplete, AdapterExtension, Srb);
            }

            StorPortEtwEvent2(adapterExtension,
                              storAddress,
                              AhciEtwEventStartIO,
                              eventDescription,
                              STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                              StorportEtwLevelInformational,
                              StorportEtwEventOpcodeInfo,
                              Srb,
                              NULL,
                              0,
                              NULL,
                              0);

            break;
        }

        case SRB_FUNCTION_IO_CONTROL:
        case SRB_FUNCTION_EXECUTE_SCSI: {
                processIO = TRUE;
                break;
            }

        case SRB_FUNCTION_DUMP_POINTERS: {
            ULONG status = STOR_STATUS_SUCCESS;
            ULONG size = 0;
            PAHCI_DUMP_CONTEXT dumpContext = NULL;
            PMINIPORT_DUMP_POINTERS dumpPointers = (PMINIPORT_DUMP_POINTERS)SrbGetDataBuffer(Srb);

            USHORT *eventDescription = L"Completed dump pointers";

            if ( (dumpPointers == NULL) ||
                 (SrbGetDataTransferLength(Srb) < RTL_SIZEOF_THROUGH_FIELD(MINIPORT_DUMP_POINTERS, MiniportPrivateDumpData)) ) {
                Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
                eventDescription = L"Invalid dump pointers request";
            } else {

                size = sizeof(AHCI_DUMP_CONTEXT);

                // allocate pool and zero the content
                status = StorPortAllocatePool(AdapterExtension,
                                              size,
                                              AHCI_POOL_TAG,
                                              (PVOID*)&dumpContext);

                if ((status != STOR_STATUS_SUCCESS) || (dumpContext == NULL)) {
                   // we cannot continue if cannot get memory for ChannelExtension.
                    Srb->SrbStatus = SRB_STATUS_ERROR;
                    eventDescription = L"Allocate dump context failed";
                } else {
                    dumpPointers->Version = DUMP_MINIPORT_VERSION_1;
                    dumpPointers->Size = sizeof(MINIPORT_DUMP_POINTERS);

                    AhciZeroMemory((PCHAR)dumpContext, size);

                    dumpContext->VendorID = adapterExtension->VendorID;
                    dumpContext->DeviceID = adapterExtension->DeviceID;
                    dumpContext->RevisionID = adapterExtension->RevisionID;
                    dumpContext->AhciBaseAddress = adapterExtension->AhciBaseAddress;
                    dumpContext->LogFlags = adapterExtension->LogFlags;
                    dumpContext->AdapterRegistryFlags = adapterExtension->RegistryFlags;

                    dumpContext->PortCount = 1;

                    dumpContext->Ports[0].PortNumber = adapterExtension->PortExtension[pathId]->PortNumber;
                    dumpContext->Ports[0].PortRegistryFlags = adapterExtension->PortExtension[pathId]->RegistryFlags;

                    //
                    // Preserve Hybrid Disk Information.
                    //
                    if (IsDeviceHybridInfoSupported(adapterExtension->PortExtension[pathId])) {
                        StorPortMoveMemory((PVOID)&dumpContext->Ports[0].HybridInfo,
                                           (PVOID)&adapterExtension->PortExtension[pathId]->DeviceExtension->HybridInfo,
                                           sizeof(GP_LOG_HYBRID_INFORMATION_HEADER));
                    }

                    //
                    // Fill telemetry collection context - shared with diskdump.sys stack
                    //
                    //dumpContext->MaxDumpLevelEnabled = 1;
                    //dumpContext->MaxDeviceDumpSize = 100;

                    AhciZeroMemory((PCHAR)dumpContext->PublicGPLogTableAddresses,sizeof(dumpContext->PublicGPLogTableAddresses));
                    dumpContext->PrivateGPLogPageAddress = 0;

                    //
                    // Override from the registry settings if passed by the storport miniport
                    // If private address is not set we will use T13 defined one . Consider always trying default one and resort to registry if it is not supported
                    //
                    if (dumpContext->PrivateGPLogPageAddress == 0) {
                        dumpContext->PrivateGPLogPageAddress = IDE_GP_LOG_CURRENT_DEVICE_INTERNAL_STATUS;
                    }

                    dumpPointers->MiniportPrivateDumpData = (PVOID)dumpContext;

                    Srb->SrbStatus = SRB_STATUS_SUCCESS;
                }
            }

            StorPortEtwChannelEvent2(adapterExtension,
                                     storAddress,
                                     (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwEventDiagnostic : StorportEtwEventOperational,
                                     AhciEtwEventStartIO,
                                     eventDescription,
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwLevelInformational : StorportEtwLevelError,
                                     StorportEtwEventOpcodeInfo,
                                     Srb,
                                     L"SrbStatus",
                                     Srb->SrbStatus,
                                     L"TransferLen",
                                     SrbGetDataTransferLength(Srb));

            StorPortNotification(RequestComplete, AdapterExtension, Srb);
            break;
        }

        case SRB_FUNCTION_FREE_DUMP_POINTERS: {
            ULONG status = STOR_STATUS_SUCCESS;
            PMINIPORT_DUMP_POINTERS dumpPointers = (PMINIPORT_DUMP_POINTERS)SrbGetDataBuffer(Srb);

            if ( (dumpPointers == NULL) ||
                 (SrbGetDataTransferLength(Srb) < RTL_SIZEOF_THROUGH_FIELD(MINIPORT_DUMP_POINTERS, MiniportPrivateDumpData)) ) {
                Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
            } else {
                status = StorPortFreePool(AdapterExtension, dumpPointers->MiniportPrivateDumpData);
                Srb->SrbStatus = (status == STOR_STATUS_SUCCESS) ? SRB_STATUS_SUCCESS : SRB_STATUS_ERROR;
            }

            StorPortEtwChannelEvent2(adapterExtension,
                                     storAddress,
                                     (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwEventDiagnostic : StorportEtwEventOperational,
                                     AhciEtwEventStartIO,
                                     L"Completed free dump pointers",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     (Srb->SrbStatus == SRB_STATUS_SUCCESS) ? StorportEtwLevelInformational : StorportEtwLevelError,
                                     StorportEtwEventOpcodeInfo,
                                     Srb,
                                     L"SrbStatus",
                                     Srb->SrbStatus,
                                     L"TransferLen",
                                     SrbGetDataTransferLength(Srb));

            StorPortNotification(RequestComplete, AdapterExtension, Srb);
            break;
        }

        default: {
            // for unsupported SRB: complete with status: SRB_STATUS_INVALID_REQUEST
            Srb->SrbStatus = SRB_STATUS_INVALID_REQUEST;
            
            StorPortEtwEvent2(adapterExtension,
                              storAddress,
                              AhciEtwEventStartIO,
                              L"Invalid request",
                              STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                              StorportEtwLevelInformational,
                              StorportEtwEventOpcodeInfo,
                              Srb,
                              L"Function",
                              function,
                              NULL,
                              0);

            StorPortNotification(RequestComplete, AdapterExtension, Srb);
            break;
        }

    } //end of switch (function)

    if (processIO) {
        // get active reference for port/device and adapter if this request
        // isn't part of D3 processing and isn't a SCSI command.
        // Storport already make sure that the Unit in active state before a SCSI command is sent to miniport.
        if ((function != SRB_FUNCTION_EXECUTE_SCSI) && (SrbGetSrbFlags(Srb) & SRB_FLAGS_D3_PROCESSING) == 0) {
            // for incoming request, port driver should make sure it's active.
            PortAcquireActiveReference(adapterExtension->PortExtension[pathId], (PSTORAGE_REQUEST_BLOCK)Srb, NULL);
        }

        AhciProcessIo(adapterExtension->PortExtension[pathId], (PSTORAGE_REQUEST_BLOCK)Srb, FALSE);
        ActivateQueue(adapterExtension->PortExtension[pathId], FALSE);
    }

    return TRUE;
}

VOID
__inline
AhciInterruptHandler(
    _In_ PAHCI_CHANNEL_EXTENSION ChannelExtension
    )
/*++
Routine Description:

    AhciInterruptHandler is the interrupt handler for a port.

    If the miniport driver requires a large amount of time to process the interrupt it must defer processing to a worker routine.
    This routine must attempt one clear the interrupt on the HBA before it returns TRUE.

NOTE:
    The following StorPort routines shall not be called from the AhciHwInterrupt routine - StorPortCompleteAllRequests and StorPortDeviceBusy.
    The miniport could however request for a worker routine and make the calls in the worker routine.

Called by:
    AhciHwInterrupt
    AhciHwMSIInterrupt

It performs:
    (overview)
    1. Prepare for handling the interrupt
    2. Handle interrupts on this port
    3. Clear port interrupt
    4. Complete outstanding commands
    5. Handle error processing if necessary

    (details)
    1.1 Verify the interrupt is for this adapter
    1.2 round robin for selecting the port to process in case of multi-ports have interrupt pending.
    1.3 Initialize Variables
    AHCI 1.1 Section 5.5.3 - 1.
    "Software determines the cause of the interrupt by reading the PxIS register.  It is possible for multiple bits to be set"
    2.1 Understand interrupts on this channel
    AHCI 1.1 Section 5.5.3 - 2.
    "Software clears appropriate bits in the PxIS register corresponding to the cause of the interrupt."
    2.1.1 Handle Fatal Errors
    Clear the PxIS Fatal Error bits
    And prep for error handling
    2.1.3 Handle non Native HotPlug Events
    Clear the PxSERR bits to clear PxIS
    2.1.3.1 Handle Serial ATA Errors (Hotplug insertion or unsolicited COMINIT)
    2.1.3.2 Handle Serial ATA Errors (Hotplug removal or PHY Power Management event)
    2.1.3.3 Handle Serial ATA Errors (everything else)
    2.1.4 Handle Datalength Mismatch Error
    Clear the PxIS Overflow bit bits
    And prep for dataLengthMisMatch handling
    2.1.15 Handle NonFatal Errors
    2.2 Handle Normal Command Completion
    3.1 Clear channel interrupt
    AHCI 1.1 Section 5.5.3 - 3.
    "Software clears the interrupt bit in IS.IPS corresponding to the port."
    4.1 Complete outstanding commands
    AHCI 1.1 Section 5.5.3 - 4.
    "If executing non-queued commands, software reads the PxCI register, and compares the current value to the list of commands previously issued by software that are still outstanding.  If executing native queued commands, software reads the PxSACT register and compares the current value to the list of commands previously issued by software.  Software completes with success any outstanding command whose corresponding bit has been cleared in the respective register. PxCI and PxSACT are volatile registers;
    software should only use their values to determine commands that have completed, not to determine which commands have previously been issued."
    4.1.1 Determine if there was an ATA error on the command completed. Error register is only valid if bit0 of Status is 1
    4.1.2 Check to see if the right amount of data transfered
    set channelExtension->Slot[i]->DataTransferLength = the amount transfered
    4.1.3 Otherwise, command completed successfully
    5.1 Handle error processing
    Until the error recovery can be completed, don't let any more IO's come through
    AHCI 1.1 Section 5.5.3 - 5.
    "If there were errors, noted in the PxIS register, software performs error recovery actions (see section 6.2.2)."
    AHCI 1.1 Section 6.2.2.1 Non-Queued Error Recovery (this may take a while, better queue a DPC)
    Complete further processing in the worker routine and enable interrupts on the channel

Affected Variables/Registers:

Return Values:
    None

--*/
{
    AHCI_INTERRUPT_STATUS   pxis;
    AHCI_INTERRUPT_STATUS   pxisMask;
    AHCI_SERIAL_ATA_STATUS  ssts;
    AHCI_SERIAL_ATA_ERROR   serr;
    AHCI_SERIAL_ATA_ERROR   serrMask;
    AHCI_COMMAND            cmd;
    ULONG                   is;
    ULONG                   ci;
    ULONG                   sact;
    ULONG                   outstanding;
    ULONG                   storStatus;
    ULONGLONG               asyncNotifyFlags;

    if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) {
        RecordExecutionHistory(ChannelExtension, 0x00000005);     //AhciInterruptHandler Enter
    }

    //1.3 Initialize Variables
    sact = 0;
    cmd.AsUlong = 0;
    ssts.AsUlong = 0;
    pxisMask.AsUlong = serrMask.AsUlong = 0;
    serr.AsUlong = 0;

    pxis.AsUlong = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong);
    
    if (pxis.IFS || pxis.HBDS || pxis.HBFS || pxis.TFES || pxis.PCS || pxis.PRCS) {
        serr.AsUlong = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong);
    }

    //2.1 Understand interrupts on this channel
    //2.1.1 Handle Fatal Errors: Interface Fatal Error Status || Host Bus Data Error Status || Host Bus Fatal Error Status || Task File Error Status
    if (pxis.IFS || pxis.HBDS || pxis.HBFS || pxis.TFES) {
        pxisMask.AsUlong = 0;
        pxisMask.IFS = pxisMask.HBDS = pxisMask.HBFS = pxisMask.TFES = 1;
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong, pxisMask.AsUlong);

        //call the correct error handling based on current hw queue workload type
        sact = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SACT);

        if ((sact == MAXULONG) && IsAdapterRemoved(ChannelExtension->AdapterExtension)) {
            // controller has been surprise removed
            StorPortEtwChannelEvent8(ChannelExtension->AdapterExtension,
                                     (ChannelExtension != NULL) ? (PSTOR_ADDRESS)&(ChannelExtension->DeviceExtension->DeviceAddress) : (NULL),
                                     StorportEtwEventOperational,
                                     AhciEtwEventHandleInterrupt,
                                     L"Adapter surprise removed",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     StorportEtwLevelWarning,
                                     StorportEtwEventOpcodeInfo,
                                     NULL,
                                     L"pxis",
                                     pxis.AsUlong,
                                     L"ssts",
                                     ssts.AsUlong,
                                     L"serr",
                                     serr.AsUlong,
                                     L"cmd",
                                     cmd.AsUlong,
                                     L"sact",
                                     sact,
                                     NULL,
                                     0,
                                     NULL,
                                     0,
                                     NULL,
                                     0);
            return;
        } else if (sact != 0) {
            //5.1 NCQ, Handle error processing
            ChannelExtension->StateFlags.CallAhciNcqErrorRecovery = 1;

            //Give NCQ one chance
            if (ChannelExtension->StateFlags.NCQ_Succeeded == 0) {
                ChannelExtension->StateFlags.NCQ_Activated = 0;
            }
        } else {
            //5.1 Non-NCQ, Handle error processing
            ChannelExtension->StateFlags.CallAhciNonQueuedErrorRecovery = 1;
        }
    }

    //2.1.2 Handle non Native HotPlug Events
    // Cold Port Detect Status
    if (pxis.CPDS) {
        pxisMask.AsUlong = 0;
        pxisMask.CPDS = 1;
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong, pxisMask.AsUlong);
        // Handle bus rescan processing
        ChannelExtension->StateFlags.CallAhciReportBusChange = 1;
    }

    if (pxis.DMPS || pxis.PCS) {
        cmd.AsUlong = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CMD.AsUlong);

        if ((cmd.AsUlong == MAXULONG) && IsAdapterRemoved(ChannelExtension->AdapterExtension)) {
            StorPortEtwChannelEvent8(ChannelExtension->AdapterExtension,
                                     (ChannelExtension != NULL) ? (PSTOR_ADDRESS)&(ChannelExtension->DeviceExtension->DeviceAddress) : (NULL),
                                     StorportEtwEventOperational,
                                     AhciEtwEventHandleInterrupt,
                                     L"Adapter surprise removed",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     StorportEtwLevelWarning,
                                     StorportEtwEventOpcodeInfo,
                                     NULL,
                                     L"pxis",
                                     pxis.AsUlong,
                                     L"ssts",
                                     ssts.AsUlong,
                                     L"serr",
                                     serr.AsUlong,
                                     L"cmd",
                                     cmd.AsUlong,
                                     L"sact",
                                     sact,
                                     NULL,
                                     0,
                                     NULL,
                                     0,
                                     NULL,
                                     0);
            return;
        }

        // Device Mechanical Presence Status
        if (pxis.DMPS) {
            pxisMask.AsUlong = 0;
            pxisMask.DMPS = 1;
            StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong, pxisMask.AsUlong);
            // Mechanical Presence Switch Attached to Port
            if (cmd.MPSP) {
                // Handle bus rescan processing
                ChannelExtension->StateFlags.CallAhciReportBusChange = 1;
            }
        }

        //2.1.3.1 Handle Serial ATA Errors (Hotplug insertion or unsolicited COMINIT)
        // Port Connect Change Status
        if (pxis.PCS) {
            //When PxSERR.DIAG.X is set to one this bit indicates a COMINIT signal was received.  This bit is reflected in the PxIS.PCS bit.
            serrMask.DIAG.X = 1;
            StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong, serrMask.AsUlong);
            // PCS = 1 could be an unsolicited COMINIT on an already detected drive. See AHCI 6.2.2.3    Recovery of Unsolicited COMINIT
            if (!IgnoreHotPlug(ChannelExtension) && (cmd.ST == 0)) {
                // Handle bus rescan processing
                ChannelExtension->StateFlags.CallAhciReportBusChange = 1;
            }
        }
    }

    //2.1.3.2 Handle Serial ATA Errors (Hotplug removal or PHY Power Management event)
    // PhyRdy Change Status
    if (pxis.PRCS) {
        //Hot plug removals are detected via the PxIS.PRCS bit that directly reflects the PxSERR.DIAG.N bit.
        //Note that PxSERR.DIAG.N is also set to '1' on insertions and during interface power management entry/exit.
        serrMask.DIAG.N = 1;
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong, serrMask.AsUlong);

        ssts.AsUlong = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SSTS.AsUlong);

        if ((ssts.AsUlong == MAXULONG) && IsAdapterRemoved(ChannelExtension->AdapterExtension)) {
            // controller has been surprise removed
            return;
        }

        if (!IgnoreHotPlug(ChannelExtension)) {
            //If a ZPODD drive has already been found and it is a ZPODD system
            if ((ChannelExtension->AdapterExtension->StateFlags.SupportsAcpiDSM == 1) &&
                IsAtapiDevice(&ChannelExtension->DeviceExtension->DeviceParameters) &&
                (ChannelExtension->DeviceExtension[0].IdentifyPacketData->SerialAtaCapabilities.SlimlineDeviceAttention)) {

                if (ssts.DET == 0) {
                    // ... (*) and there is no presence on the wire ...
                    // ... try to stop the port.
                    if (P_NotRunning(ChannelExtension, ChannelExtension->Px)) {
                        // If that succeeds, complete all outstanding commands.  CI is now cleared.
                        // This is precautionary as there shall be no IO when D3 occurred, but the miniport may always create its own commands.
                        ChannelExtension->SlotManager.CommandsToComplete = GetOccupiedSlots(ChannelExtension);
                        ChannelExtension->SlotManager.CommandsIssued = 0;
                        ChannelExtension->SlotManager.NCQueueSliceIssued = 0;
                        ChannelExtension->SlotManager.NormalQueueSliceIssued = 0;
                        ChannelExtension->SlotManager.SingleIoSliceIssued = 0;
                        ChannelExtension->SlotManager.NCQueueSlice = 0;
                        ChannelExtension->SlotManager.NormalQueueSlice = 0;
                        ChannelExtension->SlotManager.SingleIoSlice = 0;
                        ChannelExtension->SlotManager.HighPriorityAttribute = 0;
                    } else {
                        NT_ASSERT(FALSE);     // Looks like a hardware issue, will recover in P_Running_StartAttempt() when ZPODD is powered on again.
                    }
                } else if (ssts.DET == 3) {
                    // ... (*) and there is presence on the wire ...
                    // ... try to get the channel started.
                    P_Running_StartAttempt(ChannelExtension, TRUE);
                }
            } else if ((ssts.DET == 0) && (ssts.IPM == 0)) {
                // Handle bus rescan processing
                ChannelExtension->StateFlags.CallAhciReportBusChange = 1;
            }
        }
    }

    //2.1.3.3 Handle other Serial ATA Errors (everything else)
    if (serr.AsUlong > 0) {
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SERR.AsUlong, (ULONG)~0);
    }

    //2.1.4 Handle Datalength Mismatch Error
    // Overflow Status
    if (pxis.OFS) {
        pxisMask.AsUlong = 0;
        pxisMask.OFS = 1;
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong, pxisMask.AsUlong);
        //5.1 Handle error processing
        // AHCI 6.1.5 COMRESET is required by software to clean up from this serious error
        ChannelExtension->StateFlags.CallAhciReset = 1;
    }

    //2.1.5 Handle NonFatal Errors
    // Interface Non-fatal Error Status
    if (pxis.INFS) {
        pxisMask.AsUlong = 0;
        pxisMask.INFS = 1;
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong, pxisMask.AsUlong);
    }

    //2.1.6 Handle Asynchronous Notification, only ATAPI device supports this feature.
    // Set Device Bits Interrupt
    if ((pxis.SDBS == 1) && (ChannelExtension->ReceivedFIS->SetDeviceBitsFis.N) &&
        IsAtapiDevice(&ChannelExtension->DeviceExtension->DeviceParameters) &&
        IsDeviceSupportsAN(ChannelExtension->DeviceExtension->IdentifyPacketData)) {
        // Asynchronous Notification is received. Notify Port Driver.
        // This async notification could be for media status, device status or device operation events.
        // Notification failure of STOR_STATUS_BUSY is ok as it means that notifications are being coalesced
        // and this is the only async notification.
        // Notification failure of STOR_STATUS_INVALID_DEVICE_REQUEST is ok as Storport may no longer find
        // the unit as it may have been surprise removed.
        asyncNotifyFlags = (RAID_ASYNC_NOTIFY_FLAG_MEDIA_STATUS |
                            RAID_ASYNC_NOTIFY_FLAG_DEVICE_STATUS |
                            RAID_ASYNC_NOTIFY_FLAG_DEVICE_OPERATION);

        storStatus = StorPortAsyncNotificationDetected(ChannelExtension->AdapterExtension,
                                                        (PSTOR_ADDRESS)&ChannelExtension->DeviceExtension[0].DeviceAddress,
                                                        asyncNotifyFlags);

        NT_ASSERT((storStatus == STOR_STATUS_SUCCESS) ||
                  (storStatus == STOR_STATUS_BUSY) ||
                  (storStatus == STOR_STATUS_INVALID_DEVICE_REQUEST));

        // not actually use this variable.
        UNREFERENCED_PARAMETER(storStatus);
    }

    //2.3 Handle Normal Command Completion
    pxisMask.AsUlong = 0;
    // Device to Host Register FIS Interrupt
    if (pxis.DHRS) {
        pxisMask.DHRS = 1;
    }
    // PIO Setup FIS Interrupt
    if (pxis.PSS) {
        pxisMask.PSS = 1;
    }
    // DMA Setup FIS Interrupt
    if (pxis.DSS) {
        pxisMask.DSS = 1;
    }
    // Set Device Bits Interrupt
    if (pxis.SDBS) {
        pxisMask.SDBS = 1;
    }
    // Descriptor Processed (A PRD with the 'I' bit set has transferred all of its data)
    if (pxis.DPS) {
        pxisMask.DPS = 1;
    }
    if (pxisMask.AsUlong != 0) {
        StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->IS.AsUlong, pxisMask.AsUlong);
    }

    //3. error process
    if (ErrorRecoveryIsPending(ChannelExtension)) {
        // preserve taskfile for using in command completion process
        ChannelExtension->TaskFileData.AsUlong = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->TFD.AsUlong);

        if ((ChannelExtension->TaskFileData.AsUlong == MAXULONG) && IsAdapterRemoved(ChannelExtension->AdapterExtension)) {
            // controller has been surprise removed
            StorPortEtwChannelEvent8(ChannelExtension->AdapterExtension,
                                     (ChannelExtension != NULL) ? (PSTOR_ADDRESS)&(ChannelExtension->DeviceExtension->DeviceAddress) : (NULL),
                                     StorportEtwEventOperational,
                                     AhciEtwEventHandleInterrupt,
                                     L"Adapter surprise removed",
                                     STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                     StorportEtwLevelWarning,
                                     StorportEtwEventOpcodeInfo,
                                     NULL,
                                     L"pxis",
                                     pxis.AsUlong,
                                     L"ssts",
                                     ssts.AsUlong,
                                     L"serr",
                                     serr.AsUlong,
                                     L"cmd",
                                     cmd.AsUlong,
                                     L"sact",
                                     sact,
                                     L"StateFlags",
                                     *(ULONGLONG *)&(ChannelExtension->StateFlags),
                                     NULL,
                                     0,
                                     NULL,
                                     0);
            return;
        }

        AhciPortErrorRecovery(ChannelExtension);
    }
    else {
        ChannelExtension->TaskFileData.AsUlong = 0x40;
    }

    //4. Clear channel interrupt
    is = 0;
    is |= (1 << ChannelExtension->PortNumber);
    StorPortWriteRegisterUlong(ChannelExtension->AdapterExtension, ChannelExtension->AdapterExtension->IS, is);

    //5. Complete outstanding commands
    ci = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->CI);
    sact = StorPortReadRegisterUlong(ChannelExtension->AdapterExtension, &ChannelExtension->Px->SACT);

    if (((ci == MAXULONG) || (sact == MAXULONG)) && IsAdapterRemoved(ChannelExtension->AdapterExtension)) {
        // controller has been surprise removed
        StorPortEtwChannelEvent8(ChannelExtension->AdapterExtension,
                                 (ChannelExtension != NULL) ? (PSTOR_ADDRESS)&(ChannelExtension->DeviceExtension->DeviceAddress) : (NULL),
                                 StorportEtwEventOperational,
                                 AhciEtwEventHandleInterrupt,
                                 L"Adapter removed: complete cmd",
                                 STORPORT_ETW_EVENT_KEYWORD_COMMAND_TRACE,
                                 StorportEtwLevelWarning,
                                 StorportEtwEventOpcodeInfo,
                                 NULL,
                                 L"pxis",
                                 pxis.AsUlong,
                                 L"ssts",
                                 ssts.AsUlong,
                                 L"serr",
                                 serr.AsUlong,
                                 L"cmd",
                                 cmd.AsUlong,
                                 L"is",
                                 is,
                                 L"ci",
                                 ci,
                                 L"sact",
                                 sact,
                                 NULL,
                                 0);
        return;
    }

    outstanding = ci | sact;

    if ((ChannelExtension->SlotManager.CommandsIssued & (~outstanding)) > 0) {
        // all completed commands by hardware will be marked completed
        ChannelExtension->SlotManager.CommandsToComplete |= (ChannelExtension->SlotManager.CommandsIssued & ~outstanding);
        ChannelExtension->SlotManager.CommandsIssued &= outstanding;

        ChannelExtension->SlotManager.NCQueueSliceIssued &= outstanding;
        ChannelExtension->SlotManager.NormalQueueSliceIssued &= outstanding;
        ChannelExtension->SlotManager.SingleIoSliceIssued &= outstanding;

        // recording execution history for completing SRB
        RecordInterruptHistory(ChannelExtension, pxis.AsUlong, ssts.AsUlong, serr.AsUlong, ci, sact, 0x20000005);   //AhciHwInterrupt complete IO

        AhciCompleteIssuedSRBs(ChannelExtension, SRB_STATUS_SUCCESS, TRUE);
    } else {
        // recording execution history for no SRB to be completed
        RecordInterruptHistory(ChannelExtension, pxis.AsUlong, ssts.AsUlong, serr.AsUlong, ci, sact, 0x20010005);   //AhciHwInterrupt No IO completed
    }

    if (LogExecuteFullDetail(ChannelExtension->AdapterExtension->LogFlags)) {
        RecordExecutionHistory(ChannelExtension, 0x10000005);//Exit AhciInterruptHandler
    }

    return;
}

BOOLEAN
AhciHwInterrupt (
    _In_ PVOID AdapterExtension
    )
/*++

Routine Description:

    Line Interrupt handler

Arguments:

    AdapterExtension -
    MessageId -

Called by:
    external

It performs:
    1. Find the port with pending interrupt
    2. Invoke interrupt handler routine to process

Return Values:
    TRUE if the interrupt is handled.
    If the adapter (or port) did not generate interrupt, this routine should return FALSE as soon as possible.
--*/
{
    ULONG                   is;
    ULONG                   interruptPorts;
    ULONG                   i;
    UCHAR                   interruptPortCount;

    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    is = StorPortReadRegisterUlong(AdapterExtension, adapterExtension->IS);
        
    if (adapterExtension->StateFlags.Removed) {
        return FALSE;
    }

    interruptPorts = (is & adapterExtension->PortImplemented);

    //
    // Verify the interrupt is for this adapter
    //
    if (interruptPorts == 0) {
        return FALSE;
    }

    interruptPortCount = NumberOfSetBits(interruptPorts);

    //
    // In case of multi-ports have interrupt pending, using round robin to choose a port handle its interrupt.
    // After this routine exits, other pending interrupts will trigger interrupt again.
    //
    if (interruptPortCount > 1) {
        i = (adapterExtension->LastInterruptedPort + 1) % (adapterExtension->HighestPort + 1);
    } else {
        i = adapterExtension->LastInterruptedPort;
    }

    do {
        if ( ((interruptPorts & (1 << i)) != 0) && IsPortStartCapable(adapterExtension->PortExtension[i]) ) {
            break;
        }
        i = (i + 1) % (adapterExtension->HighestPort + 1);

    } while (i != adapterExtension->LastInterruptedPort);

    if ( (i == adapterExtension->LastInterruptedPort) &&
         (((interruptPorts & (1 << i)) == 0) || !IsPortStartCapable(adapterExtension->PortExtension[i])) ) {
        //
        // interrupt is not for this adapter
        //
        return FALSE;
    }

    adapterExtension->LastInterruptedPort = i;

    AhciInterruptHandler(adapterExtension->PortExtension[i]);

    return TRUE;
}

BOOLEAN
AhciHwMSIInterrupt (
    _In_ PVOID AdapterExtension,
    _In_ ULONG MessageId
    )
/*++

Routine Description:

    MSI Interrupt handler

Arguments:

    AdapterExtension -
    MessageId -

Return Value:

    BOOLEAN - TRUE indicates it's interrupt for AHCI adapter

--*/
{
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    if (adapterExtension->StateFlags.InterruptMessagePerPort == 0) {
        //
        // In case message ID is shared by ports, call legacy ISR to determine if it's a valid AHCI interrupt.
        //
        return AhciHwInterrupt(AdapterExtension);
    }

    //
    // Per AHCI 1.3, Message ID is the same value as the port number.
    //
    AhciInterruptHandler(adapterExtension->PortExtension[MessageId]);

    return TRUE;
}


VOID
AhciHwTracingEnabled (
    _In_ PVOID AdapterExtension,
    _In_ BOOLEAN Enabled
    )
{
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;

    adapterExtension->TracingEnabled = Enabled;

    return;
}

SCSI_UNIT_CONTROL_STATUS
AhciHwUnitControl (
    _In_ PVOID AdapterExtension,
    _In_ SCSI_UNIT_CONTROL_TYPE ControlType,
    _In_ PVOID Parameters
    )
/*++

Routine Description:

    This is a generic routine to allow for special unit control activities.
    It mainly handles device start irp for a specific device.

Arguments:

    AdapterExtension - Pointer to the device extension for adapter.

    ControlType - Specifies the type of call being made through this routine.

    Parameters - Pointer to parameters needed for this control type (optional).

Return Value:

    SCSI_UNIT_CONTROL_STATUS - currently either:
        ScsiUnitControlSuccess = 0,
        ScsiUnitControlUnsuccessful

--*/

{
    SCSI_UNIT_CONTROL_STATUS status = ScsiUnitControlSuccess;
    PAHCI_ADAPTER_EXTENSION adapterExtension = (PAHCI_ADAPTER_EXTENSION)AdapterExtension;
    PSCSI_SUPPORTED_CONTROL_TYPE_LIST controlTypeList;

    switch (ControlType) {
        // determine which control types (routines) are supported
        case ScsiQuerySupportedUnitControlTypes: {
            // get pointer to control type list
            controlTypeList = (PSCSI_SUPPORTED_CONTROL_TYPE_LIST)Parameters;

            // Report ScsiQuerySupportedUnitControlTypes, ScsiUnitStart and ScsiUnitPower are supported.
            if (ScsiQuerySupportedControlTypes < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiQuerySupportedUnitControlTypes] = TRUE;
            }
            if (ScsiUnitStart < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiUnitStart] = TRUE;
            }
            if (ScsiUnitPower < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiUnitPower] = TRUE;
            }
            if (ScsiUnitPoFxPowerInfo < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiUnitPoFxPowerInfo] = TRUE;
            }
            if (ScsiUnitPoFxPowerActive < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiUnitPoFxPowerActive] = TRUE;
            }
            if (ScsiUnitPoFxPowerSetFState < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiUnitPoFxPowerSetFState] = TRUE;
            }
            if (ScsiUnitSurpriseRemoval < controlTypeList->MaxControlType) {
                controlTypeList->SupportedTypeList[ScsiUnitSurpriseRemoval] = TRUE;
            }
            status = ScsiUnitControlSuccess;
            break;
        }

        // ScsiUnitStart is called during Storport processes IRP_MN_START_DEVICE
        case ScsiUnitStart: {
            PSTOR_ADDR_BTL8 storAddrBtl8 = (PSTOR_ADDR_BTL8)Parameters;
            // this is the PnP call IRP_MN_START_DEVICE for the LUN. Need to do device initialization work.
            if (IsPortValid(adapterExtension, storAddrBtl8->Path)) {
                AhciDeviceStart(adapterExtension->PortExtension[storAddrBtl8->Path]);
                status = ScsiUnitControlSuccess;
            } else {
                status = ScsiUnitControlUnsuccessful;
            }
            break;
        }

        // ScsiUnitPower is called during Storport processes power Irp for the unit
        case ScsiUnitPower: {
            PSTOR_UNIT_CONTROL_POWER unitControlPower = (PSTOR_UNIT_CONTROL_POWER)Parameters;
            PSTOR_ADDR_BTL8 storAddrBtl8 = (PSTOR_ADDR_BTL8)unitControlPower->Address;
            // this is the power call for the LUN. Need to power up or power down the device as necessary.
            if (IsPortValid(adapterExtension, storAddrBtl8->Path)) {
                PAHCI_CHANNEL_EXTENSION channelExtension = adapterExtension->PortExtension[storAddrBtl8->Path];

                StorPortDebugPrint(3, "StorAHCI - LPM: SystemIoBusNumber:%d Port:%02d - %s\n", 
                    adapterExtension->SystemIoBusNumber, storAddrBtl8->Path, unitControlPower->PowerState == StorPowerDeviceD0 ? "D0" : "D3");
                
                if (unitControlPower->PowerState == StorPowerDeviceD0) {
                    AhciPortPowerUp(channelExtension);
                    channelExtension->DevicePowerState = StorPowerDeviceD0;
                    status = ScsiUnitControlSuccess;

                } else if (unitControlPower->PowerState == StorPowerDeviceD3) {

                    AhciPortPowerDown(channelExtension);
                    channelExtension->DevicePowerState = StorPowerDeviceD3;
                    status = ScsiUnitControlSuccess;
                } else {
                    NT_ASSERT(FALSE);
                    status = ScsiUnitControlUnsuccessful;
                }
            } else {
                status = ScsiUnitControlUnsuccessful;
            }
            break;
        }

        case ScsiUnitPoFxPowerInfo: {
            PSTOR_POFX_UNIT_POWER_INFO unitPowerInfo = (PSTOR_POFX_UNIT_POWER_INFO)Parameters;
            PSTOR_ADDR_BTL8 storAddrBtl8 = (PSTOR_ADDR_BTL8)unitPowerInfo->Header.Address;
            BOOLEAN d3ColdEnabled = FALSE;

            if (IsPortValid(adapterExtension, storAddrBtl8->Path)) {
                ULONG storStatus = STOR_STATUS_SUCCESS;
                ULONG bufferLength = STOR_POFX_DEVICE_V3_SIZE + STOR_POFX_COMPONENT_V2_SIZE + STOR_POFX_COMPONENT_IDLE_STATE_SIZE;
                PAHCI_CHANNEL_EXTENSION channelExtension = adapterExtension->PortExtension[storAddrBtl8->Path];
                BOOLEAN reportF1State = FALSE;

                //
                // IdlePowerEnabled == TRUE indicates this unit is being
                // registered for runtime power management.  However, the call
                // to StorPortInitializePoFxPower() must succeed before the unit
                // is truly and successfully registered.
                //
                if (unitPowerInfo->IdlePowerEnabled) {

                    //
                    // Don't register more than once.
                    //
                    if (channelExtension->StateFlags.PoFxEnabled) {
                        status = ScsiUnitControlUnsuccessful;
                        break;
                    }

                    if (reportF1State) {
                        // the buffer should be big enough to contain one more F-State (F1)
                        bufferLength += STOR_POFX_COMPONENT_IDLE_STATE_SIZE;
                    }

                    // allocate STOR_POFX_DEVICE data structure for Port, initialize the structure and register for runtime power management.
                    if (channelExtension->PoFxDevice == NULL) {
                        storStatus = StorPortAllocatePool(AdapterExtension,
                                                          bufferLength,
                                                          AHCI_POOL_TAG,
                                                          (PVOID*)&channelExtension->PoFxDevice);
                    }

                    if (storStatus == STOR_STATUS_SUCCESS) {
                        PSTOR_POFX_COMPONENT_V2 component = (PSTOR_POFX_COMPONENT_V2)&(channelExtension->PoFxDevice->Components[0]);

                        AhciZeroMemory((PCHAR)channelExtension->PoFxDevice, bufferLength);

                        channelExtension->PoFxDevice->Version = STOR_POFX_DEVICE_VERSION_V3;
                        channelExtension->PoFxDevice->Size = STOR_POFX_DEVICE_V3_SIZE;
                        channelExtension->PoFxDevice->ComponentCount = 1;

                        //
                        // The dump version of StorAHCI is not able to bring the
                        // unit out of D3Cold.
                        //
                        channelExtension->PoFxDevice->Flags |= STOR_POFX_DEVICE_FLAG_NO_DUMP_ACTIVE;

                        //
                        // Disable idle debouncing if it's possible for us to
                        // initiate a Slumber transition.
                        // We set the Slumber timer when we get the Idle
                        // condition callback and cancel it when we get the
                        // Active condition callback.  The slumber timer is
                        // typically set to 100ms and idle debouncing can add
                        // a bias of up to 1s.  Significantly delaying Slumber
                        // entry can have a negative impact on power so we
                        // disable idle debouncing.
                        // CAP.SSC indicates if the HBA supports Slumber.
                        // CAP.SALP indicates if the HBA can automatically
                        // transition to Partial when idle, which is needed for
                        // us to subsequently initiate a Slumber transition.
                        //
                        if ((channelExtension->AdapterExtension->CAP.SSC) &&
                            (channelExtension->AdapterExtension->CAP.SALP) &&
                            IsDeviceSupportsHIPM(channelExtension->DeviceExtension[0].IdentifyDeviceData)) {
                            channelExtension->PoFxDevice->Flags |= STOR_POFX_DEVICE_FLAG_NO_IDLE_DEBOUNCE;
                        }

                        //
                        // If this is a drive with rotational media then enable adaptive D3 idle timeout, else if
                        // this is a SSD on an AOAC system, set the idle timeout flag and provide a minimum unit idle timeout.
                        //
                        if (DeviceIncursSeekPenalty(channelExtension)) {
                            channelExtension->PoFxDevice->Flags |= STOR_POFX_DEVICE_FLAG_ADAPTIVE_D3_IDLE_TIMEOUT;
                        } else if (IsReceivingSystemPowerHints(adapterExtension) && DeviceIncursNoSeekPenalty(channelExtension)) {
                            channelExtension->PoFxDevice->Flags |= STOR_POFX_DEVICE_FLAG_IDLE_TIMEOUT;
                        }

                        component->Version = STOR_POFX_COMPONENT_VERSION_V2;
                        component->Size = STOR_POFX_COMPONENT_V2_SIZE;
                        component->FStateCount = reportF1State ? 2 : 1;
                        component->DeepestWakeableFState = 0;
                        component->DeepestAdapterPowerRequiredFState = 0;
                        component->DeepestCrashDumpReadyFState = 0;
                        component->Id = STORPORT_POFX_LUN_GUID;

                        // F0 State
                        component->FStates[0].Version = STOR_POFX_COMPONENT_IDLE_STATE_VERSION_V1;
                        component->FStates[0].Size = STOR_POFX_COMPONENT_IDLE_STATE_SIZE;
                        component->FStates[0].TransitionLatency = 0;
                        component->FStates[0].ResidencyRequirement = 0;
                        component->FStates[0].NominalPower = STOR_POFX_UNKNOWN_POWER;

                        // registry runtime power management for Unit
                        storStatus = StorPortInitializePoFxPower(AdapterExtension,
                                                                 (PSTOR_ADDRESS)storAddrBtl8,
                                                                 (PSTOR_POFX_DEVICE)channelExtension->PoFxDevice,
                                                                 &d3ColdEnabled);

                        channelExtension->StateFlags.D3ColdEnabled = d3ColdEnabled;
                    }

                    if (storStatus != STOR_STATUS_SUCCESS) {
                        StorPortFreePool(AdapterExtension, channelExtension->PoFxDevice);
                        channelExtension->PoFxDevice = NULL;
                        channelExtension->StateFlags.PoFxEnabled = FALSE;
                        channelExtension->StateFlags.PoFxActive = FALSE;

                        status = ScsiUnitControlUnsuccessful;
                    } else {
                        channelExtension->StateFlags.PoFxEnabled = TRUE;
                        channelExtension->StateFlags.PoFxActive = TRUE;

                    }

                    channelExtension->PoFxFState = 0;

                } else {
                    //
                    // IdlePowerEnabled = FALSE, which means this unit is no
                    // longer registered for runtime power management.
                    //
                    if (channelExtension->PoFxDevice) {
                        StorPortFreePool(AdapterExtension, channelExtension->PoFxDevice);
                        channelExtension->PoFxDevice = NULL;
                    }
                    channelExtension->StateFlags.PoFxEnabled = FALSE;
                    channelExtension->StateFlags.PoFxActive = FALSE;

                    status = ScsiUnitControlSuccess;
                }
            } else {
                status = ScsiUnitControlUnsuccessful;
            }
            break;
        }

        case ScsiUnitPoFxPowerActive: {
            PSTOR_POFX_ACTIVE_CONTEXT activeContext = (PSTOR_POFX_ACTIVE_CONTEXT)Parameters;
            PSTOR_ADDR_BTL8 storAddrBtl8 = (PSTOR_ADDR_BTL8)activeContext->Header.Address;

            if (IsPortValid(adapterExtension, storAddrBtl8->Path) && PortPoFxEnabled(adapterExtension->PortExtension[storAddrBtl8->Path])) {
                PAHCI_CHANNEL_EXTENSION channelExtension = adapterExtension->PortExtension[storAddrBtl8->Path];

                channelExtension->StateFlags.PoFxActive = activeContext->Active ? 1 : 0;

                StorPortDebugPrint(3, "StorAHCI - LPM: SystemIoBusNumber:%d Port:%02d - %s\n", 
                    adapterExtension->SystemIoBusNumber, channelExtension->PortNumber, activeContext->Active ? "ACTIVE" : "IDLE");

                if (activeContext->Active) {
                    ULONG busChangePending;
                    ULONG restorePreservedSettingsPending;

                    //
                    // Cancel the Slumber timer.
                    //
                    StorPortRequestTimer(channelExtension->AdapterExtension,
                                         channelExtension->WorkerTimer,
                                         AhciAutoPartialToSlumber,
                                         channelExtension,
                                         0, 0);
                    StorPortDebugPrint(3, "StorAHCI - LPM: SystemIoBusNumber:%d Port:%02d - Transit into Slumber from Partial - Canceled\n", 
                        adapterExtension->SystemIoBusNumber, channelExtension->PortNumber);

                    busChangePending = InterlockedBitTestAndReset((LONG*)&channelExtension->PoFxPendingWork, 1);  //BusChange is at bit 1
                    restorePreservedSettingsPending = InterlockedBitTestAndReset((LONG*)&channelExtension->PoFxPendingWork, 0);  //RestorePreservedSettings is at bit 0

                    //perform pending Unit PoFx work
                    if (busChangePending == 1) {
                        PortBusChangeProcess(channelExtension);
                    }

                    if (restorePreservedSettingsPending == 1) {
                        // continue on processing RestorePreservedSettings

                        RecordExecutionHistory(channelExtension, 0x10050041); // RestorePreservedSettings continue in Unit Active

                        // Start the first command
                        IssuePreservedSettingCommands(channelExtension, NULL);

                        // Starts processing the command. Only need to do the first command if it exists. all others will be done by processing completion routine.
                        if (channelExtension->Local.Srb.SrbExtension != NULL) {
                            PAHCI_SRB_EXTENSION srbExtension = GetSrbExtension((PSTORAGE_REQUEST_BLOCK)&channelExtension->Local.Srb);
                            if (srbExtension->AtaFunction != 0) {
                                AhciProcessIo(channelExtension, (PSTORAGE_REQUEST_BLOCK)&channelExtension->Local.Srb, FALSE);
                                ActivateQueue(channelExtension, FALSE);
                            }
                        }
                    }
                } else {
                    //
                    // Attempt to start the Slumber timer now that we're idle.
                    // Use a tolerable delay of 20ms to allow coalescing with
                    // other timers.
                    //
                    if (PartialToSlumberTransitionIsAllowed(channelExtension, NULL)) {

                        status = StorPortRequestTimer(channelExtension->AdapterExtension,
                                                            channelExtension->WorkerTimer,
                                                            AhciAutoPartialToSlumber,
                                                            channelExtension,
                                                            channelExtension->AutoPartialToSlumberInterval * 1000, 20000);

                        if (status == STOR_STATUS_SUCCESS) {
                            StorPortDebugPrint(3, "StorAHCI - LPM: SystemIoBusNumber:%d Port:%02d - Transit into Slumber from Partial - Scheduled\n", 
                                adapterExtension->SystemIoBusNumber, channelExtension->PortNumber);
                        }
                    }
                    
                }
            } else {
                status = ScsiUnitControlUnsuccessful;
            }

            break;
        }

        case ScsiUnitPoFxPowerSetFState: {
            PSTOR_POFX_FSTATE_CONTEXT fStateContext = (PSTOR_POFX_FSTATE_CONTEXT)Parameters;
            PSTOR_ADDR_BTL8 storAddrBtl8 = (PSTOR_ADDR_BTL8)fStateContext->Header.Address;
            PAHCI_CHANNEL_EXTENSION channelExtension = adapterExtension->PortExtension[storAddrBtl8->Path];

            if (IsPortValid(adapterExtension, storAddrBtl8->Path) && PortPoFxEnabled(channelExtension)) {

                StorPortDebugPrint(3, "StorAHCI - LPM: SystemIoBusNumber:%d Port:%02d - Transition from F%u to F%u\n", 
                    adapterExtension->SystemIoBusNumber, channelExtension->PortNumber, channelExtension->PoFxFState, fStateContext->FState);

                channelExtension->PoFxFState = (UCHAR)fStateContext->FState;                

                if (fStateContext->FState == 0) {
                } else if (fStateContext->FState == 1) {
                } else {
                    NT_ASSERT(FALSE);
                    status = ScsiUnitControlUnsuccessful;
                }

            } else {
                status = ScsiUnitControlUnsuccessful;
            }

            break;
        }

        //
        // ScsiUnitSurpriseRemoval is called when Storport processes the surprise removal IRP for the unit.
        //
        case ScsiUnitSurpriseRemoval: {
            PSTOR_ADDR_BTL8 storAddrBtl8 = (PSTOR_ADDR_BTL8)Parameters;
            STOR_LOCK_HANDLE lockhandle = {InterruptLock, 0};

            if (IsPortValid(adapterExtension, storAddrBtl8->Path)) {

                PAHCI_CHANNEL_EXTENSION channelExtension = adapterExtension->PortExtension[storAddrBtl8->Path];
                AhciInterruptSpinlockAcquire(adapterExtension, channelExtension->PortNumber, &lockhandle);

                //
                // Complete all issued commands.
                //
                channelExtension->SlotManager.CommandsToComplete = channelExtension->SlotManager.CommandsIssued;
                channelExtension->SlotManager.CommandsIssued = 0;
                channelExtension->SlotManager.NCQueueSliceIssued = 0;
                channelExtension->SlotManager.NormalQueueSliceIssued = 0;
                channelExtension->SlotManager.SingleIoSliceIssued = 0;
                channelExtension->SlotManager.HighPriorityAttribute &= ~channelExtension->SlotManager.CommandsToComplete;
                
                AhciCompleteIssuedSRBs(channelExtension, SRB_STATUS_NO_DEVICE, TRUE); 

                //
                // Complete all other commands miniport owns for this device.
                //
                AhciPortFailAllIos(channelExtension, SRB_STATUS_NO_DEVICE, TRUE);

                AhciInterruptSpinlockRelease(adapterExtension, channelExtension->PortNumber, &lockhandle);

                ++(channelExtension->TotalCountSurpriseRemove);
                AhciTelemetryLogResetErrorRecovery(channelExtension,
                                                  (PSTOR_ADDRESS)&(channelExtension->DeviceExtension->DeviceAddress),
                                                  AhciTelemetryEventIdSurpriseRemove,
                                                  "ScsiUnitSurpriseRemoval",
                                                  AHCI_TELEMETRY_FLAG_NOT_SUPPRESS_LOGGING,
                                                  NULL,
                                                  0
                                                  );
            }

            break;
        }

        default:
            status = ScsiUnitControlUnsuccessful;
            break;

    } // end of switch

    return status;
}

#if _MSC_VER >= 1200
#pragma warning(pop)
#else
#pragma warning(default:4152)
#pragma warning(default:4214)
#pragma warning(default:4201)
#endif