Merge branch 'develop' into user/daspr/kmodsamplefix

1 files changed, 1001 insertions, 0 deletions

diff --git a/network/netadaptercx/netvadapterlibrary/code/enl.cpp b/network/netadaptercx/netvadapterlibrary/code/enl.cpp
new file mode 100644
index 00000000..709c5a1e
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/enl.cpp
@@ -0,0 +1,1001 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#include "pch.hpp"
+#include <stdlib.h>
+#include "netvadapter.h"
+#include "rxqueue.h"
+#include "txqueue.h"
+#include "trace.h"
+#include "memory.h"
+#include "rtl\KLockHolder.h"
+#include "enl.tmh"
+
+///////////////////////////////////////////////////////////////////////////////
+// BEGIN Generic execution engine thread lib                                 //
+///////////////////////////////////////////////////////////////////////////////
+
+ENL_MLINK NetvEnlMLink[MAX_ADAPTER_COUNT / 2];
+
+/*++
+The iteration routine performs one full pass over all input queues and
+any internal queues (that might be holding previous items waiting to be
+processed).
+--*/
+
+static
+SIZE_T
+CopyTxPacketDataToBuffer(
+    _Out_writes_bytes_(BufferLength) PUCHAR BufferDest,
+    _In_ NET_RING_PACKET_ITERATOR const * Iterator,
+    _In_ NET_EXTENSION const * VirtualAddressExtension,
+    _In_ SIZE_T BufferLength)
+{
+    SIZE_T bytesCopied = 0;
+
+    for (NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(Iterator);
+         NetFragmentIteratorHasAny(&fi) && (BufferLength > 0);
+         NetFragmentIteratorAdvance(&fi))
+    {
+        NET_FRAGMENT const * fragment = NetFragmentIteratorGetFragment(&fi);
+        NET_FRAGMENT_VIRTUAL_ADDRESS const * virtualAddress =
+            NetExtensionGetFragmentVirtualAddress(VirtualAddressExtension, NetFragmentIteratorGetIndex(&fi));
+
+        UCHAR const * pPacketData = (UCHAR const *)virtualAddress->VirtualAddress + fragment->Offset;
+        SIZE_T bytesToCopy =
+            (BufferLength < (SIZE_T) fragment->ValidLength) ? BufferLength : (SIZE_T) fragment->ValidLength;
+        RtlCopyMemory(BufferDest, pPacketData, bytesToCopy);
+
+        bytesCopied += bytesToCopy;
+        BufferDest += bytesToCopy;
+        BufferLength -= bytesToCopy;
+    }
+
+    return bytesCopied;
+}
+
+VOID
+EnlpAffinitizeThread(
+    _In_ ULONG ProcIndex,
+    _In_ ULONG IdealNode
+    )
+{
+    LogInformation(FLAG_DRIVER, L"ProcIndex=%u", ProcIndex);
+    NTSTATUS status;
+    PROCESSOR_NUMBER procNumber = { 0 };
+    GROUP_AFFINITY affinity = { 0 };
+
+    if (ProcIndex == 999)
+    {
+        // No affinity at all
+        return;
+    }
+
+    if (ProcIndex == 1000 || (ProcIndex >= 1001 && ProcIndex <= 1004))
+    {
+        KeQueryNodeActiveAffinity((USHORT)IdealNode, &affinity, NULL);
+
+        if (ProcIndex == 1000)
+        {
+            // Affinitize to all the procs in the Node
+            NOTHING;
+        }
+        else
+        {
+            // Affinitize to the highest numbered proc in the Node for 1001,
+            // next highest numbered proc for 1002, ...
+            ULONG index;
+#ifdef _WIN64
+            NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse64(&index, (ULONG64)affinity.Mask));
+            affinity.Mask = (KAFFINITY)((ULONG64)1 << (index - (ProcIndex - 1001)));
+#else
+            NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse(&index, (ULONG)affinity.Mask));
+            affinity.Mask = (KAFFINITY)((ULONG)1 << (index - (ProcIndex - 1001)));
+#endif
+        }
+    }
+    else
+    {
+        // Affinitize the specifically requested proc
+        status = KeGetProcessorNumberFromIndex(ProcIndex, &procNumber);
+        NT_FRE_ASSERTMSG("Bad ProcIndex", NT_SUCCESS(status));
+
+        affinity.Group = procNumber.Group;
+        affinity.Mask = AFFINITY_MASK(procNumber.Number);
+    }
+
+    EnlThreadSetAffinity(&affinity, NULL);
+}
+
+ENL_START_ROUTINE EnlpThreadRoutine;
+
+NTSTATUS
+EnlpStartThread(
+    _In_ ULONG ProcIndex,
+    _In_ ULONG IdealNode,
+    _In_ ENLP_ITERATION_ROUTINE* IterationRoutine,
+    _In_ PVOID IterationContext,
+    _In_opt_ KAutoEvent* ArmWaitEvent,
+    _Out_ ENLP_THREAD_STATE* EnlThread
+    )
+{
+    LogInformation(FLAG_DRIVER, L"ProcIndex=%u EnlThread=%p", ProcIndex, EnlThread);
+
+    EnlThread->PauseRequested = TRUE;
+    EnlThread->StopRequested = FALSE;
+    EnlThread->IterationRoutine = IterationRoutine;
+    EnlThread->IterationContext = IterationContext;
+    EnlThread->ArmWaitEvent = ArmWaitEvent;
+
+    EnlThread->ProcIndex = ProcIndex;
+    EnlThread->IdealNode = IdealNode;
+
+    RETURN_IF_NOT_STATUS_SUCCESS(
+        EnlThreadCreate(EnlpThreadRoutine, EnlThread, EnlThread->Thread));
+
+    EnlThreadSetPriority(EnlThread->Thread, 15);
+
+    RETURN_STATUS_SUCCESS();
+}
+
+BOOLEAN
+EnlpIsThreadPaused(
+    _In_ CONST ENLP_THREAD_STATE* EnlThread
+    )
+{
+    return EnlThread->PauseRequested;
+}
+
+VOID
+EnlpPauseThread(
+    _Inout_ ENLP_THREAD_STATE* EnlThread
+    )
+{
+    LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread);
+    if (!EnlpIsThreadPaused(EnlThread))
+    {
+        WriteBooleanNoFence(&EnlThread->PauseRequested, TRUE);
+        if (EnlThread->ArmWaitEvent != NULL)
+        {
+            EnlThread->ArmWaitEvent->Set();
+        }
+        EnlThread->PausingEvent.Wait();
+    }
+}
+
+VOID
+EnlpResumeThread(
+    _Inout_ ENLP_THREAD_STATE* EnlThread
+    )
+{
+    LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread);
+    if (EnlpIsThreadPaused(EnlThread))
+    {
+        WriteBooleanNoFence(&EnlThread->PauseRequested, FALSE);
+        KeMemoryBarrier();
+        EnlThread->ResumeEvent.Set();
+    }
+}
+
+VOID
+EnlpStopThread(
+    _Inout_ ENLP_THREAD_STATE* EnlThread
+    )
+{
+    LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread);
+    if (EnlThread->Thread != NULL)
+    {
+        WriteBooleanNoFence(&EnlThread->StopRequested, TRUE);
+        EnlpResumeThread(EnlThread); // in case thread was paused
+        EnlThreadWaitForTermination(EnlThread->Thread);
+        EnlThread->Thread.reset();
+    }
+}
+
+ENL_THREAD_ROUTINE_RETURN
+EnlpThreadRoutine(
+    _In_ PVOID Context
+    )
+{
+    ENLP_THREAD_STATE* enlThread = (ENLP_THREAD_STATE*)Context;
+
+    //
+    // Affinitize the thread
+    //
+    EnlpAffinitizeThread(enlThread->ProcIndex, enlThread->IdealNode);
+
+    //
+    // ENL thread starts at paused state.
+    //
+    enlThread->ResumeEvent.Wait();
+
+    for (;;)
+    {
+
+        if (ReadBooleanNoFence(&enlThread->PauseRequested))
+        {
+            enlThread->PausingEvent.Set();
+            enlThread->ResumeEvent.Wait();
+        }
+
+        if (ReadBooleanNoFence(&enlThread->StopRequested))
+        {
+            break;
+        }
+
+        enlThread->IterationRoutine(enlThread->IterationContext);
+    }
+
+    return ENL_THREAD_ROUTINE_RETURN();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// END Generic execution engine thread lib                                   //
+///////////////////////////////////////////////////////////////////////////////
+
+BOOLEAN
+enlpCheckAndArmQueue(
+    _Inout_ ENLP_QUEUE* Q
+    )
+{
+    BOOLEAN armed = FALSE;
+    auto ringBuffer = Q->Queue->GetPacketRing();
+    // If Q is still empty, take the lock, and if still empty under lock, then
+    // arm it.
+    if (ringBuffer->BeginIndex == ringBuffer->EndIndex)
+    {
+        WdfSpinLockAcquire(Q->Spinlock);
+
+        if (ringBuffer->BeginIndex == ringBuffer->EndIndex)
+        {
+            armed = TRUE;
+            Q->Armed = TRUE;
+        }
+
+        WdfSpinLockRelease(Q->Spinlock);
+    }
+
+    return armed;
+}
+
+VOID
+enlpArmAndWait(
+    _Inout_ ENLP_LINK* EnlLink
+    )
+{
+    ULONG pi, ci;
+
+    for (pi = 0; pi < ENLP_PORT_COUNT; pi++)
+    {
+        ENLP_PORT* port = &EnlLink->Ports[pi];
+
+        for (ci = 0; ci < port->TxQueueCount; ci++)
+        {
+            if (!enlpCheckAndArmQueue(port->TxQueue))
+            {
+                return;
+            }
+        }
+    }
+
+    EnlLink->ArmWaitEvent.Wait();
+}
+
+VOID
+EnlpIterationRoutine(
+    _In_ PVOID IterationContext
+    )
+{
+    auto enlLink = reinterpret_cast<ENLP_LINK *>(IterationContext);
+    bool emptyTx = true;
+    //
+    // Drain up to TX_BATCH_COUNT items from all tx queues first.
+    //
+
+    for (size_t pi = 0U; pi < ENLP_PORT_COUNT; pi++)
+    {
+        auto txport = &enlLink->Ports[pi];
+        auto rxport = &enlLink->Ports[pi ^ 0x1]; // 0 -> 1, 1 -> 0
+
+        for (size_t ci = 0U; ci < txport->TxQueueCount; ci++)
+        {
+            auto txq = &txport->TxQueue[ci];
+            if (txq->State != Started)
+                continue;
+
+            NET_RING_PACKET_ITERATOR txPi = {
+                txq->Queue->m_rings, nullptr, txq->QueueNext, txq->QueueEnd
+            };
+
+            while (NetPacketIteratorHasAny(&txPi))
+            {
+                bool rxDrop = TRUE;
+                auto txPacket = NetPacketIteratorGetPacket(&txPi);
+
+                emptyTx = FALSE;
+
+                if (enlLink->ArmedForWake)
+                {
+                    // Save wake packet and trigger a wake signal, any other frames queued after the wake
+                    // packet will be dropped
+                    enlLink->WakeFrameSize = CopyTxPacketDataToBuffer(
+                        &enlLink->WakeFrame[0],
+                        &txPi,
+                        &txq->TxQueue->VirtualAddressExtension,
+                        sizeof(enlLink->WakeFrame));
+#if _KERNEL_MODE
+                    enlLink->EvtWakeSignal(enlLink->WakeSignalContext);
+#endif
+
+                    // Make sure to disarm wake, otherwise this thread might overwrite the original wake frame
+                    EnlDisarmWake(enlLink);
+                }
+
+                if (rxport->RxQueueCount > 0)
+                {
+                    //TODO: Currently does 1:1 mapping between Tx and Rx. Need to set up indirection table
+                    auto rxq = &rxport->RxQueue[ci];
+
+                    if (rxq->State == Started)
+                    {
+                        NET_RING_FRAGMENT_ITERATOR rxFi = {
+                            rxq->Queue->m_rings, nullptr, rxq->QueueNext, rxq->QueueEnd
+                        };
+
+                        auto rxPi = NetRingGetPostPackets(rxq->Queue->m_rings);
+
+                        if (NetFragmentIteratorHasAny(&rxFi) && NetPacketIteratorHasAny(&rxPi))
+                        {
+                            rxDrop = FALSE;
+
+                            auto fragment = NetFragmentIteratorGetFragment(&rxFi);
+                            BYTE* fragmentBuffer = nullptr;
+
+                            auto const rxVirtualAddress =
+                                NetExtensionGetFragmentVirtualAddress(
+                                    &rxq->RxQueue->VirtualAddressExtension,
+                                    NetFragmentIteratorGetIndex(&rxFi));
+
+                            fragmentBuffer = static_cast<BYTE *>(rxVirtualAddress->VirtualAddress);
+
+                            fragment->Offset = 0;
+                            fragment->ValidLength =
+                                CopyTxPacketDataToBuffer(
+                                    fragmentBuffer,
+                                    &txPi,
+                                    &txq->TxQueue->VirtualAddressExtension,
+                                    static_cast<SIZE_T>(fragment->Capacity));
+
+                            auto rxPacket = NetPacketIteratorGetPacket(&rxPi);
+                            rxPacket->FragmentIndex =  NetFragmentIteratorGetIndex(&rxFi);
+                            rxPacket->FragmentCount = 1;
+
+                            if (enlLink->InitializePacketLayout)
+                            {
+                                rxPacket->Layout = txPacket->Layout;
+                            }
+                            else
+                            {
+                                rxPacket->Layout = {};
+                            }
+
+                            if (rxq->RxQueue->RxXSumExtension.Enabled)
+                            {
+                                auto RxXSum = NetExtensionGetPacketChecksum(
+                                    &rxq->RxQueue->RxXSumExtension,
+                                    NetPacketIteratorGetIndex(&rxPi));
+                                RxXSum->Layer2 = NetPacketRxChecksumEvaluationNotChecked;
+                                RxXSum->Layer3 = NetPacketRxChecksumEvaluationValid;
+                                RxXSum->Layer4 = NetPacketRxChecksumEvaluationValid;
+                            }
+
+                            if (rxq->RxQueue->UdpRscExtension.Enabled &&
+                                txq->TxQueue->UsoExtension.Enabled)
+                            {
+                                auto txUso = NetExtensionGetPacketGso(
+                                    &txq->TxQueue->UsoExtension,
+                                    NetPacketIteratorGetIndex(&txPi));
+                                if (txPacket->Layout.Layer4Type == NetPacketLayer4TypeUdp &&
+                                    txUso->UDP.Mss > 0)
+                                {
+                                    if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv6NoExtensions)
+                                    {
+                                        UINT16* ipv6PayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 4);
+                                        *ipv6PayloadLength = _byteswap_ushort(
+                                            (USHORT)(fragment->ValidLength -
+                                            (txPacket->Layout.Layer2HeaderLength + txPacket->Layout.Layer3HeaderLength)));
+                                    }
+                                    else if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv4NoOptions)
+                                    {
+                                        UINT16* ipv4TotalLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 2);
+                                        *ipv4TotalLength = _byteswap_ushort((USHORT)(fragment->ValidLength - txPacket->Layout.Layer2HeaderLength ));
+                                        NT_FRE_ASSERT(*ipv4TotalLength > 0);
+                                    }
+
+                                    UINT16* udpPayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength +
+                                                                         txPacket->Layout.Layer3HeaderLength + 4);
+                                    *udpPayloadLength = _byteswap_ushort(
+                                        (USHORT)(fragment->ValidLength -
+                                        (txPacket->Layout.Layer3HeaderLength + txPacket->Layout.Layer2HeaderLength)));
+
+                                    // Set the checksum to zero
+                                    UINT16* udpChecksum = udpPayloadLength + 1;
+                                    *udpChecksum = 0;
+
+                                    auto rxUro = NetExtensionGetPacketRsc(
+                                        &rxq->RxQueue->UdpRscExtension,
+                                        NetPacketIteratorGetIndex(&rxPi));
+                                    rxUro->UDP.CoalescedSegmentSize = (UINT16)txUso->UDP.Mss;
+                                    rxUro->UDP.CoalescedSegmentCount =
+                                        (UINT16)((fragment->ValidLength - txPacket->Layout.Layer3HeaderLength - txPacket->Layout.Layer4HeaderLength
+                                         + txUso->UDP.Mss - 1) / txUso->UDP.Mss);
+                                    NT_FRE_ASSERT(rxUro->UDP.CoalescedSegmentCount > 1);
+                                    NT_FRE_ASSERT(*udpPayloadLength >= 8);
+                                }
+                            }
+
+                            // prevent any reordering the tx/rx completion flag
+                            KeMemoryBarrier();
+
+                            // Use Scratch field as completion flag for the rx fragment
+                            fragment->Scratch = 1;
+                            rxPacket->Scratch = 1;
+
+                            NetFragmentIteratorAdvance(&rxFi);
+                            NetPacketIteratorAdvance(&rxPi);
+
+                            rxq->QueueNext = NetFragmentIteratorGetIndex(&rxFi);
+                            NetPacketIteratorSet(&rxPi);
+
+                            if (rxq->Notify)
+                            {
+                                rxport->Interrupt(rxq->Queue->m_handle, rxq->TxRx);
+                            }
+                        }
+                    }
+                }
+
+                if (rxDrop)
+                {
+                    // TODO - add rxdrop stat
+                }
+
+                // Use Scratch field as completion flag for the tx packet
+                txPacket->Scratch = 1;
+                NetPacketIteratorAdvance(&txPi);
+            }
+
+            if (!emptyTx)
+            {
+                if (txq->Notify)
+                {
+                    txport->Interrupt(txq->Queue->m_handle, txq->TxRx);
+                }
+
+                // Store the next index so that the EnlThread knows which packet to start from in the next iteration
+                txq->QueueNext = NetPacketIteratorGetIndex(&txPi);
+            }
+        }
+    }
+
+    ULONG64 ts;
+
+    if (emptyTx)
+    {
+        if (enlLink->Poll == FALSE)
+        {
+            enlpArmAndWait(enlLink);
+        }
+
+        ts = ReadTimeStampCounter();
+        enlLink->EmptyTicks += (ts - enlLink->Ts);
+    }
+    else
+    {
+        ts = ReadTimeStampCounter();
+        enlLink->BusyTicks += (ts - enlLink->Ts);
+    }
+
+    enlLink->Ts = ts;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+ENLP_QUEUE *
+EnlCreateQueue(
+    _In_ NETPACKETQUEUE Queue,
+    _In_ BOOLEAN Tx
+    )
+{
+    ENLP_LINK* enlLink;
+    ENLP_PORT* port;
+    ENLP_QUEUE* enlQueue;
+    NTSTATUS status;
+
+    if (Tx) // Tx
+    {
+        NetvTxQueue* netvTxQueue = NetvTxQueueGetContext(Queue);
+        enlLink = NetvEnlMLink[netvTxQueue->m_adapter.EnlIndex].LinkHandle[0];
+        port = &enlLink->Ports[netvTxQueue->m_adapter.EnlPortIndex];
+        enlQueue = &port->TxQueue[0]; // Change to accommodate multiple Queues
+        enlQueue->Queue = netvTxQueue;
+        enlQueue->TxQueue = netvTxQueue;
+        enlQueue->State = Stopped;
+        enlQueue->ArmWaitEvent = &enlLink->ArmWaitEvent;
+        enlQueue->QueueNext = 0;
+        enlQueue->QueueEnd = 0;
+        enlQueue->TxRx = TX;
+        enlQueue->EnlPortHandle = port;
+        port->TxQueueCount++;
+        LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu TxQueue=%p",
+            &netvTxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvTxQueue);
+    }
+    else // Rx
+    {
+        NetvRxQueue* netvRxQueue = NetvRxQueueGetContext(Queue);
+        enlLink = NetvEnlMLink[netvRxQueue->m_adapter.EnlIndex].LinkHandle[0];
+        port = &enlLink->Ports[netvRxQueue->m_adapter.EnlPortIndex];
+        enlQueue = &port->RxQueue[0]; // Change to accommodate multiple Queues
+        enlQueue->Queue = netvRxQueue;
+        enlQueue->RxQueue = netvRxQueue;
+        enlQueue->State = Stopped;
+        enlQueue->QueueNext = 0;
+        enlQueue->QueueEnd = 0;
+        enlQueue->TxRx = RX;
+        enlQueue->EnlPortHandle = port;
+        port->RxQueueCount++;
+        LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu RxQueue=%p",
+            &netvRxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvRxQueue);
+    }
+
+    // Create WDF spinlock for the queue, parent to the queue's WDF handle
+    WDF_OBJECT_ATTRIBUTES attributes;
+    WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
+    attributes.ParentObject = enlQueue->Queue->m_handle;
+    status = WdfSpinLockCreate(&attributes, &enlQueue->Spinlock);
+    NT_ASSERT(NT_SUCCESS(status));
+
+
+    EnlpResumeThread(&enlLink->EnlThread);
+
+    return enlQueue;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlDestroyQueue(
+    _In_ ENLP_QUEUE * Queue,
+    _In_ BOOLEAN Tx
+    )
+{
+    LogInformation(FLAG_DRIVER, L"Queue=%p", Queue);
+    ENLP_PORT* port = Queue->EnlPortHandle;
+
+    if (Tx)
+    {
+        port->TxQueueCount--;
+    }
+    else
+    {
+        port->RxQueueCount--;
+    }
+
+    Queue->ArmWaitEvent = nullptr;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlRingDoorBell(
+    _In_ ENLP_QUEUE * Queue,
+    _In_ ULONG EndIndex
+    )
+{
+    InterlockedExchange((volatile LONG *)&Queue->QueueEnd, (LONG)EndIndex);
+
+    WdfSpinLockAcquire(Queue->Spinlock);
+    
+    if (Queue->Armed)
+    {
+        NT_ASSERT(Queue->ArmWaitEvent != NULL);
+        Queue->Armed = FALSE;
+        WdfSpinLockRelease(Queue->Spinlock);
+        Queue->ArmWaitEvent->Set();
+        return;
+    }
+
+    WdfSpinLockRelease(Queue->Spinlock);
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlArmInterrupt(
+    _In_ ENLP_QUEUE * Queue,
+    _In_ BOOLEAN notificationEnabled
+    )
+{
+    Queue->Notify = notificationEnabled;
+
+    // TODO: InterlockedExchange reduces the throughput of the ENL from about 500MB/s to 8MB/s. -> why
+    //InterlockedExchange((volatile LONG *)&Queue->Notify, (LONG)notificationEnabled);
+}
+
+_Use_decl_annotations_
+VOID
+EnlIndicateQueueState(
+    ENLP_QUEUE * Queue,
+    ENL_QUEUE_STATE State
+    )
+{
+    Queue->State = State;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlCreateLink(
+    _In_ ULONG ProcessorIndex,
+    _In_ ULONG IdealNode,
+    _In_ BOOLEAN Poll,
+    _In_ BOOLEAN InitializePacketLayout,
+    _Out_ ENLP_LINK ** EnlLink
+    )
+{
+    LogInformation(FLAG_DRIVER, L"ProcessorIndex=%u", ProcessorIndex);
+
+    auto enlLink = wil::make_unique_nothrow<ENLP_LINK>();
+    RETURN_NTSTATUS_IF(
+        STATUS_INSUFFICIENT_RESOURCES,
+        ! enlLink);
+
+    enlLink->Ts = ReadTimeStampCounter();
+    enlLink->Poll = Poll;
+    enlLink->InitializePacketLayout = InitializePacketLayout;
+
+    RETURN_IF_NOT_STATUS_SUCCESS(
+        EnlpStartThread(
+            ProcessorIndex,
+            IdealNode,
+            EnlpIterationRoutine,
+            enlLink.get(),
+            enlLink->Poll ? NULL : &enlLink->ArmWaitEvent,
+            &enlLink->EnlThread));
+
+    *EnlLink = enlLink.release();
+
+    RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlIsPortActive(
+    _In_ ENLP_LINK * EnlLink,
+    _In_ ULONG PortIndex
+    )
+{
+    ENLP_PORT* port = &EnlLink->Ports[PortIndex];
+    ENLP_QUEUE* txq = &port->TxQueue[0];
+    NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT);
+
+    return (txq->Queue == nullptr) ? FALSE : TRUE;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlIsLinkActive(
+    _In_ ENLP_LINK * EnlLink
+    )
+{
+    return (EnlIsPortActive(EnlLink, 0) || EnlIsPortActive(EnlLink, 1));
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlActivateLinkPort(
+    _In_ ENLP_LINK * EnlLink,
+    _In_ ULONG PortIndex,
+    _In_ ENL_INTERRUPT_ROUTINE Interrupt,
+    _In_ PVOID PortContext
+    )
+{
+    ENLP_PORT* port = &EnlLink->Ports[PortIndex];
+
+    LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue);
+
+    NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, PortIndex));
+    NT_FRE_ASSERT(port->TxQueueCount == 0);
+    NT_FRE_ASSERT(port->RxQueueCount == 0);
+
+    port->Interrupt = Interrupt;
+    port->PortContext = PortContext;
+
+    RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlDeactivateLinkPort(
+    _In_ ENLP_LINK * EnlLink,
+    _In_ ULONG PortIndex
+    )
+{
+    NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT);
+
+    ENLP_PORT* port = &EnlLink->Ports[PortIndex];
+    LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue);
+
+    NT_FRE_ASSERT(EnlIsPortActive(EnlLink, PortIndex));
+
+    KLockThisExclusive(EnlLink->Lock);
+    NT_FRE_ASSERT(!EnlpIsThreadPaused(&EnlLink->EnlThread));
+    EnlpPauseThread(&EnlLink->EnlThread);
+
+#if _KERNEL_MODE
+    KeFlushQueuedDpcs();
+#endif
+
+    port->PortContext = NULL;
+
+    //Clears reference to only first queue - Change for all queues
+    port->TxQueue[0].Queue = nullptr;
+    port->RxQueue[0].Queue = nullptr;
+
+
+    // As long as there's one port with active queues, the EnlThread will run.
+    for (ULONG i = 0; i < ENLP_PORT_COUNT; i++)
+    {
+        if (EnlIsPortActive(EnlLink, i))
+        {
+            EnlpResumeThread(&EnlLink->EnlThread);
+            break;
+        }
+    }
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlCloseLink(
+    _In_ ENLP_LINK * EnlLink
+    )
+{
+    ULONG i;
+
+    for (i = 0; i < ENLP_PORT_COUNT; i++)
+    {
+        NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, i));
+    }
+
+    EnlpStopThread(&EnlLink->EnlThread);
+
+    delete EnlLink;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//  Multi link wrapper APIs                                                  //
+///////////////////////////////////////////////////////////////////////////////
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlMCreateLink(
+    _In_range_(1, ENL_MLINK_MAX)ULONG LinkCount,
+    _In_reads_(LinkCount) ULONG ProcessorIndex,
+    _In_ BOOLEAN Poll,
+    _Out_ ENL_MLINK* EnlMLink
+    )
+{
+    NTSTATUS status = STATUS_SUCCESS;
+    ULONG i = 0;
+
+    RtlZeroMemory(EnlMLink, sizeof(*EnlMLink));
+
+    if (LinkCount < 1 || LinkCount > ENL_MLINK_MAX ||
+        (LinkCount & (LinkCount - 1)) != 0)
+    {
+        status = STATUS_REQUEST_NOT_ACCEPTED;
+        goto exit;
+    }
+
+    for (i = 0; i < LinkCount; i++)
+    {
+#ifdef _KERNEL_MODE
+        const BOOLEAN initializePacketLayout = TRUE;
+#else
+        const BOOLEAN initializePacketLayout = FALSE;
+#endif
+        status = EnlCreateLink(ProcessorIndex, 0, Poll, initializePacketLayout, &EnlMLink->LinkHandle[i]);
+
+        if (!NT_SUCCESS(status))
+        {
+            goto exit;
+        }
+    }
+
+    EnlMLink->LinkCount = LinkCount;
+
+exit:
+
+    if (!NT_SUCCESS(status))
+    {
+        for (; i > 0; i--)
+        {
+            EnlCloseLink(EnlMLink->LinkHandle[i - 1]);
+            EnlMLink->LinkHandle[i - 1] = NULL;
+        }
+    }
+    RETURN_NTSTATUS_IF(
+        status,
+        status != STATUS_SUCCESS);
+
+    RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void EnlSetPdoWakeSignalCallback(
+    _In_ ENLP_LINK * EnlLink,
+    _In_ EVT_ENLP_PDO_WAKE_SIGNAL* evtPdoWakeSignal,
+    _In_ void* Context)
+{
+    EnlLink->EvtWakeSignal = evtPdoWakeSignal;
+    EnlLink->WakeSignalContext = Context;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlArmWake(
+    _In_ ENLP_LINK * EnlLink
+)
+{
+    EnlLink->ArmedForWake = TRUE;
+    RtlZeroMemory(&EnlLink->WakeFrame[0], sizeof(EnlLink->WakeFrame));
+    EnlLink->WakeFrameSize = 0;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlDisarmWake(
+    _In_ ENLP_LINK * EnlLink
+)
+{
+    EnlLink->ArmedForWake = FALSE;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+size_t
+EnlCopyWakeFrame(
+    _In_ ENLP_LINK * EnlLink,
+    _Out_writes_bytes_(BufferSize) unsigned char * Buffer,
+    _In_ size_t BufferSize
+)
+{
+    if (EnlLink->WakeFrameSize == 0)
+    {
+        // There was no wake
+        return 0;
+    }
+
+    if (BufferSize < EnlLink->WakeFrameSize)
+    {
+        // Wake frame is larger than what we can indicate
+        return 0;
+    }
+
+    RtlCopyMemory(Buffer, &EnlLink->WakeFrame[0], EnlLink->WakeFrameSize);
+    auto const wakeFrameSize = EnlLink->WakeFrameSize;
+
+    // Make sure to erase the wake frame, since the network interface might have
+    // multiple receive queues
+    RtlZeroMemory(&EnlLink->WakeFrame[0], EnlLink->WakeFrameSize);
+    EnlLink->WakeFrameSize = 0;
+
+    return wakeFrameSize;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlMIsPortActive(
+    _In_ CONST ENL_MLINK* EnlMLink,
+    _In_ ULONG PortIndex
+    )
+{
+    ULONG i;
+
+    BOOLEAN result = EnlIsPortActive(EnlMLink->LinkHandle[0], PortIndex);
+
+    for (i = 1; i < EnlMLink->LinkCount; i++)
+    {
+        NT_FRE_ASSERT(EnlIsPortActive(EnlMLink->LinkHandle[i], PortIndex) == result);
+    }
+
+    return result;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlMIsLinkActive(
+    _In_ CONST ENL_MLINK* EnlMLink
+    )
+{
+    ULONG i;
+
+    BOOLEAN result = EnlIsLinkActive(EnlMLink->LinkHandle[0]);
+
+    for (i = 1; i < EnlMLink->LinkCount; i++)
+    {
+        NT_FRE_ASSERT(EnlIsLinkActive(EnlMLink->LinkHandle[i]) == result);
+    }
+
+    return result;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlMActivateLinkPort(
+    _In_ CONST ENL_MLINK* EnlMLink,
+    _In_ ULONG PortIndex,
+    _In_ ENL_INTERRUPT_ROUTINE Interrupt,
+    _In_ PVOID PortContext
+    )
+{
+    LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex);
+
+    NTSTATUS status = STATUS_SUCCESS;
+    ULONG i;
+
+    for (i = 0; i < EnlMLink->LinkCount; i++)
+    {
+        status = EnlActivateLinkPort(
+            EnlMLink->LinkHandle[i],
+            PortIndex,
+            Interrupt,
+            PortContext);
+
+        if (!NT_SUCCESS(status))
+        {
+            break;
+        }
+    }
+
+    if (!NT_SUCCESS(status))
+    {
+        for (; i > 0; i--)
+        {
+            EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex);
+        }
+    }
+
+    RETURN_NTSTATUS_IF(status,
+        status != STATUS_SUCCESS);
+
+    RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlMDeactivateLinkPort(
+    _In_ CONST ENL_MLINK* EnlMLink,
+    _In_ ULONG PortIndex
+    )
+{
+    LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex);
+
+    ULONG i;
+
+    for (i = 0; i < EnlMLink->LinkCount; i++)
+    {
+        EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex);
+    }
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlMCloseLink(
+    _Inout_ ENL_MLINK* EnlMLink
+    )
+{
+    LogInformation(FLAG_DRIVER, L"EnlMLink=%p", EnlMLink);
+
+    ULONG i;
+
+    for (i = 0; i < EnlMLink->LinkCount; i++)
+    {
+        EnlCloseLink(EnlMLink->LinkHandle[i]);
+        EnlMLink->LinkHandle[i] = NULL;
+    }
+
+    EnlMLink->LinkCount = 0;
+}