diff options
Diffstat (limited to 'network/netadaptercx/netvadapterlibrary/code/enl.cpp')
| -rw-r--r-- | network/netadaptercx/netvadapterlibrary/code/enl.cpp | 1001 |
1 files changed, 1001 insertions, 0 deletions
diff --git a/network/netadaptercx/netvadapterlibrary/code/enl.cpp b/network/netadaptercx/netvadapterlibrary/code/enl.cpp new file mode 100644 index 00000000..709c5a1e --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/enl.cpp @@ -0,0 +1,1001 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#include "pch.hpp" +#include <stdlib.h> +#include "netvadapter.h" +#include "rxqueue.h" +#include "txqueue.h" +#include "trace.h" +#include "memory.h" +#include "rtl\KLockHolder.h" +#include "enl.tmh" + +/////////////////////////////////////////////////////////////////////////////// +// BEGIN Generic execution engine thread lib // +/////////////////////////////////////////////////////////////////////////////// + +ENL_MLINK NetvEnlMLink[MAX_ADAPTER_COUNT / 2]; + +/*++ +The iteration routine performs one full pass over all input queues and +any internal queues (that might be holding previous items waiting to be +processed). +--*/ + +static +SIZE_T +CopyTxPacketDataToBuffer( + _Out_writes_bytes_(BufferLength) PUCHAR BufferDest, + _In_ NET_RING_PACKET_ITERATOR const * Iterator, + _In_ NET_EXTENSION const * VirtualAddressExtension, + _In_ SIZE_T BufferLength) +{ + SIZE_T bytesCopied = 0; + + for (NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(Iterator); + NetFragmentIteratorHasAny(&fi) && (BufferLength > 0); + NetFragmentIteratorAdvance(&fi)) + { + NET_FRAGMENT const * fragment = NetFragmentIteratorGetFragment(&fi); + NET_FRAGMENT_VIRTUAL_ADDRESS const * virtualAddress = + NetExtensionGetFragmentVirtualAddress(VirtualAddressExtension, NetFragmentIteratorGetIndex(&fi)); + + UCHAR const * pPacketData = (UCHAR const *)virtualAddress->VirtualAddress + fragment->Offset; + SIZE_T bytesToCopy = + (BufferLength < (SIZE_T) fragment->ValidLength) ? BufferLength : (SIZE_T) fragment->ValidLength; + RtlCopyMemory(BufferDest, pPacketData, bytesToCopy); + + bytesCopied += bytesToCopy; + BufferDest += bytesToCopy; + BufferLength -= bytesToCopy; + } + + return bytesCopied; +} + +VOID +EnlpAffinitizeThread( + _In_ ULONG ProcIndex, + _In_ ULONG IdealNode + ) +{ + LogInformation(FLAG_DRIVER, L"ProcIndex=%u", ProcIndex); + NTSTATUS status; + PROCESSOR_NUMBER procNumber = { 0 }; + GROUP_AFFINITY affinity = { 0 }; + + if (ProcIndex == 999) + { + // No affinity at all + return; + } + + if (ProcIndex == 1000 || (ProcIndex >= 1001 && ProcIndex <= 1004)) + { + KeQueryNodeActiveAffinity((USHORT)IdealNode, &affinity, NULL); + + if (ProcIndex == 1000) + { + // Affinitize to all the procs in the Node + NOTHING; + } + else + { + // Affinitize to the highest numbered proc in the Node for 1001, + // next highest numbered proc for 1002, ... + ULONG index; +#ifdef _WIN64 + NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse64(&index, (ULONG64)affinity.Mask)); + affinity.Mask = (KAFFINITY)((ULONG64)1 << (index - (ProcIndex - 1001))); +#else + NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse(&index, (ULONG)affinity.Mask)); + affinity.Mask = (KAFFINITY)((ULONG)1 << (index - (ProcIndex - 1001))); +#endif + } + } + else + { + // Affinitize the specifically requested proc + status = KeGetProcessorNumberFromIndex(ProcIndex, &procNumber); + NT_FRE_ASSERTMSG("Bad ProcIndex", NT_SUCCESS(status)); + + affinity.Group = procNumber.Group; + affinity.Mask = AFFINITY_MASK(procNumber.Number); + } + + EnlThreadSetAffinity(&affinity, NULL); +} + +ENL_START_ROUTINE EnlpThreadRoutine; + +NTSTATUS +EnlpStartThread( + _In_ ULONG ProcIndex, + _In_ ULONG IdealNode, + _In_ ENLP_ITERATION_ROUTINE* IterationRoutine, + _In_ PVOID IterationContext, + _In_opt_ KAutoEvent* ArmWaitEvent, + _Out_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"ProcIndex=%u EnlThread=%p", ProcIndex, EnlThread); + + EnlThread->PauseRequested = TRUE; + EnlThread->StopRequested = FALSE; + EnlThread->IterationRoutine = IterationRoutine; + EnlThread->IterationContext = IterationContext; + EnlThread->ArmWaitEvent = ArmWaitEvent; + + EnlThread->ProcIndex = ProcIndex; + EnlThread->IdealNode = IdealNode; + + RETURN_IF_NOT_STATUS_SUCCESS( + EnlThreadCreate(EnlpThreadRoutine, EnlThread, EnlThread->Thread)); + + EnlThreadSetPriority(EnlThread->Thread, 15); + + RETURN_STATUS_SUCCESS(); +} + +BOOLEAN +EnlpIsThreadPaused( + _In_ CONST ENLP_THREAD_STATE* EnlThread + ) +{ + return EnlThread->PauseRequested; +} + +VOID +EnlpPauseThread( + _Inout_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread); + if (!EnlpIsThreadPaused(EnlThread)) + { + WriteBooleanNoFence(&EnlThread->PauseRequested, TRUE); + if (EnlThread->ArmWaitEvent != NULL) + { + EnlThread->ArmWaitEvent->Set(); + } + EnlThread->PausingEvent.Wait(); + } +} + +VOID +EnlpResumeThread( + _Inout_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread); + if (EnlpIsThreadPaused(EnlThread)) + { + WriteBooleanNoFence(&EnlThread->PauseRequested, FALSE); + KeMemoryBarrier(); + EnlThread->ResumeEvent.Set(); + } +} + +VOID +EnlpStopThread( + _Inout_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread); + if (EnlThread->Thread != NULL) + { + WriteBooleanNoFence(&EnlThread->StopRequested, TRUE); + EnlpResumeThread(EnlThread); // in case thread was paused + EnlThreadWaitForTermination(EnlThread->Thread); + EnlThread->Thread.reset(); + } +} + +ENL_THREAD_ROUTINE_RETURN +EnlpThreadRoutine( + _In_ PVOID Context + ) +{ + ENLP_THREAD_STATE* enlThread = (ENLP_THREAD_STATE*)Context; + + // + // Affinitize the thread + // + EnlpAffinitizeThread(enlThread->ProcIndex, enlThread->IdealNode); + + // + // ENL thread starts at paused state. + // + enlThread->ResumeEvent.Wait(); + + for (;;) + { + + if (ReadBooleanNoFence(&enlThread->PauseRequested)) + { + enlThread->PausingEvent.Set(); + enlThread->ResumeEvent.Wait(); + } + + if (ReadBooleanNoFence(&enlThread->StopRequested)) + { + break; + } + + enlThread->IterationRoutine(enlThread->IterationContext); + } + + return ENL_THREAD_ROUTINE_RETURN(); +} + +/////////////////////////////////////////////////////////////////////////////// +// END Generic execution engine thread lib // +/////////////////////////////////////////////////////////////////////////////// + +BOOLEAN +enlpCheckAndArmQueue( + _Inout_ ENLP_QUEUE* Q + ) +{ + BOOLEAN armed = FALSE; + auto ringBuffer = Q->Queue->GetPacketRing(); + // If Q is still empty, take the lock, and if still empty under lock, then + // arm it. + if (ringBuffer->BeginIndex == ringBuffer->EndIndex) + { + WdfSpinLockAcquire(Q->Spinlock); + + if (ringBuffer->BeginIndex == ringBuffer->EndIndex) + { + armed = TRUE; + Q->Armed = TRUE; + } + + WdfSpinLockRelease(Q->Spinlock); + } + + return armed; +} + +VOID +enlpArmAndWait( + _Inout_ ENLP_LINK* EnlLink + ) +{ + ULONG pi, ci; + + for (pi = 0; pi < ENLP_PORT_COUNT; pi++) + { + ENLP_PORT* port = &EnlLink->Ports[pi]; + + for (ci = 0; ci < port->TxQueueCount; ci++) + { + if (!enlpCheckAndArmQueue(port->TxQueue)) + { + return; + } + } + } + + EnlLink->ArmWaitEvent.Wait(); +} + +VOID +EnlpIterationRoutine( + _In_ PVOID IterationContext + ) +{ + auto enlLink = reinterpret_cast<ENLP_LINK *>(IterationContext); + bool emptyTx = true; + // + // Drain up to TX_BATCH_COUNT items from all tx queues first. + // + + for (size_t pi = 0U; pi < ENLP_PORT_COUNT; pi++) + { + auto txport = &enlLink->Ports[pi]; + auto rxport = &enlLink->Ports[pi ^ 0x1]; // 0 -> 1, 1 -> 0 + + for (size_t ci = 0U; ci < txport->TxQueueCount; ci++) + { + auto txq = &txport->TxQueue[ci]; + if (txq->State != Started) + continue; + + NET_RING_PACKET_ITERATOR txPi = { + txq->Queue->m_rings, nullptr, txq->QueueNext, txq->QueueEnd + }; + + while (NetPacketIteratorHasAny(&txPi)) + { + bool rxDrop = TRUE; + auto txPacket = NetPacketIteratorGetPacket(&txPi); + + emptyTx = FALSE; + + if (enlLink->ArmedForWake) + { + // Save wake packet and trigger a wake signal, any other frames queued after the wake + // packet will be dropped + enlLink->WakeFrameSize = CopyTxPacketDataToBuffer( + &enlLink->WakeFrame[0], + &txPi, + &txq->TxQueue->VirtualAddressExtension, + sizeof(enlLink->WakeFrame)); +#if _KERNEL_MODE + enlLink->EvtWakeSignal(enlLink->WakeSignalContext); +#endif + + // Make sure to disarm wake, otherwise this thread might overwrite the original wake frame + EnlDisarmWake(enlLink); + } + + if (rxport->RxQueueCount > 0) + { + //TODO: Currently does 1:1 mapping between Tx and Rx. Need to set up indirection table + auto rxq = &rxport->RxQueue[ci]; + + if (rxq->State == Started) + { + NET_RING_FRAGMENT_ITERATOR rxFi = { + rxq->Queue->m_rings, nullptr, rxq->QueueNext, rxq->QueueEnd + }; + + auto rxPi = NetRingGetPostPackets(rxq->Queue->m_rings); + + if (NetFragmentIteratorHasAny(&rxFi) && NetPacketIteratorHasAny(&rxPi)) + { + rxDrop = FALSE; + + auto fragment = NetFragmentIteratorGetFragment(&rxFi); + BYTE* fragmentBuffer = nullptr; + + auto const rxVirtualAddress = + NetExtensionGetFragmentVirtualAddress( + &rxq->RxQueue->VirtualAddressExtension, + NetFragmentIteratorGetIndex(&rxFi)); + + fragmentBuffer = static_cast<BYTE *>(rxVirtualAddress->VirtualAddress); + + fragment->Offset = 0; + fragment->ValidLength = + CopyTxPacketDataToBuffer( + fragmentBuffer, + &txPi, + &txq->TxQueue->VirtualAddressExtension, + static_cast<SIZE_T>(fragment->Capacity)); + + auto rxPacket = NetPacketIteratorGetPacket(&rxPi); + rxPacket->FragmentIndex = NetFragmentIteratorGetIndex(&rxFi); + rxPacket->FragmentCount = 1; + + if (enlLink->InitializePacketLayout) + { + rxPacket->Layout = txPacket->Layout; + } + else + { + rxPacket->Layout = {}; + } + + if (rxq->RxQueue->RxXSumExtension.Enabled) + { + auto RxXSum = NetExtensionGetPacketChecksum( + &rxq->RxQueue->RxXSumExtension, + NetPacketIteratorGetIndex(&rxPi)); + RxXSum->Layer2 = NetPacketRxChecksumEvaluationNotChecked; + RxXSum->Layer3 = NetPacketRxChecksumEvaluationValid; + RxXSum->Layer4 = NetPacketRxChecksumEvaluationValid; + } + + if (rxq->RxQueue->UdpRscExtension.Enabled && + txq->TxQueue->UsoExtension.Enabled) + { + auto txUso = NetExtensionGetPacketGso( + &txq->TxQueue->UsoExtension, + NetPacketIteratorGetIndex(&txPi)); + if (txPacket->Layout.Layer4Type == NetPacketLayer4TypeUdp && + txUso->UDP.Mss > 0) + { + if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv6NoExtensions) + { + UINT16* ipv6PayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 4); + *ipv6PayloadLength = _byteswap_ushort( + (USHORT)(fragment->ValidLength - + (txPacket->Layout.Layer2HeaderLength + txPacket->Layout.Layer3HeaderLength))); + } + else if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv4NoOptions) + { + UINT16* ipv4TotalLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 2); + *ipv4TotalLength = _byteswap_ushort((USHORT)(fragment->ValidLength - txPacket->Layout.Layer2HeaderLength )); + NT_FRE_ASSERT(*ipv4TotalLength > 0); + } + + UINT16* udpPayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + + txPacket->Layout.Layer3HeaderLength + 4); + *udpPayloadLength = _byteswap_ushort( + (USHORT)(fragment->ValidLength - + (txPacket->Layout.Layer3HeaderLength + txPacket->Layout.Layer2HeaderLength))); + + // Set the checksum to zero + UINT16* udpChecksum = udpPayloadLength + 1; + *udpChecksum = 0; + + auto rxUro = NetExtensionGetPacketRsc( + &rxq->RxQueue->UdpRscExtension, + NetPacketIteratorGetIndex(&rxPi)); + rxUro->UDP.CoalescedSegmentSize = (UINT16)txUso->UDP.Mss; + rxUro->UDP.CoalescedSegmentCount = + (UINT16)((fragment->ValidLength - txPacket->Layout.Layer3HeaderLength - txPacket->Layout.Layer4HeaderLength + + txUso->UDP.Mss - 1) / txUso->UDP.Mss); + NT_FRE_ASSERT(rxUro->UDP.CoalescedSegmentCount > 1); + NT_FRE_ASSERT(*udpPayloadLength >= 8); + } + } + + // prevent any reordering the tx/rx completion flag + KeMemoryBarrier(); + + // Use Scratch field as completion flag for the rx fragment + fragment->Scratch = 1; + rxPacket->Scratch = 1; + + NetFragmentIteratorAdvance(&rxFi); + NetPacketIteratorAdvance(&rxPi); + + rxq->QueueNext = NetFragmentIteratorGetIndex(&rxFi); + NetPacketIteratorSet(&rxPi); + + if (rxq->Notify) + { + rxport->Interrupt(rxq->Queue->m_handle, rxq->TxRx); + } + } + } + } + + if (rxDrop) + { + // TODO - add rxdrop stat + } + + // Use Scratch field as completion flag for the tx packet + txPacket->Scratch = 1; + NetPacketIteratorAdvance(&txPi); + } + + if (!emptyTx) + { + if (txq->Notify) + { + txport->Interrupt(txq->Queue->m_handle, txq->TxRx); + } + + // Store the next index so that the EnlThread knows which packet to start from in the next iteration + txq->QueueNext = NetPacketIteratorGetIndex(&txPi); + } + } + } + + ULONG64 ts; + + if (emptyTx) + { + if (enlLink->Poll == FALSE) + { + enlpArmAndWait(enlLink); + } + + ts = ReadTimeStampCounter(); + enlLink->EmptyTicks += (ts - enlLink->Ts); + } + else + { + ts = ReadTimeStampCounter(); + enlLink->BusyTicks += (ts - enlLink->Ts); + } + + enlLink->Ts = ts; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +ENLP_QUEUE * +EnlCreateQueue( + _In_ NETPACKETQUEUE Queue, + _In_ BOOLEAN Tx + ) +{ + ENLP_LINK* enlLink; + ENLP_PORT* port; + ENLP_QUEUE* enlQueue; + NTSTATUS status; + + if (Tx) // Tx + { + NetvTxQueue* netvTxQueue = NetvTxQueueGetContext(Queue); + enlLink = NetvEnlMLink[netvTxQueue->m_adapter.EnlIndex].LinkHandle[0]; + port = &enlLink->Ports[netvTxQueue->m_adapter.EnlPortIndex]; + enlQueue = &port->TxQueue[0]; // Change to accommodate multiple Queues + enlQueue->Queue = netvTxQueue; + enlQueue->TxQueue = netvTxQueue; + enlQueue->State = Stopped; + enlQueue->ArmWaitEvent = &enlLink->ArmWaitEvent; + enlQueue->QueueNext = 0; + enlQueue->QueueEnd = 0; + enlQueue->TxRx = TX; + enlQueue->EnlPortHandle = port; + port->TxQueueCount++; + LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu TxQueue=%p", + &netvTxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvTxQueue); + } + else // Rx + { + NetvRxQueue* netvRxQueue = NetvRxQueueGetContext(Queue); + enlLink = NetvEnlMLink[netvRxQueue->m_adapter.EnlIndex].LinkHandle[0]; + port = &enlLink->Ports[netvRxQueue->m_adapter.EnlPortIndex]; + enlQueue = &port->RxQueue[0]; // Change to accommodate multiple Queues + enlQueue->Queue = netvRxQueue; + enlQueue->RxQueue = netvRxQueue; + enlQueue->State = Stopped; + enlQueue->QueueNext = 0; + enlQueue->QueueEnd = 0; + enlQueue->TxRx = RX; + enlQueue->EnlPortHandle = port; + port->RxQueueCount++; + LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu RxQueue=%p", + &netvRxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvRxQueue); + } + + // Create WDF spinlock for the queue, parent to the queue's WDF handle + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT(&attributes); + attributes.ParentObject = enlQueue->Queue->m_handle; + status = WdfSpinLockCreate(&attributes, &enlQueue->Spinlock); + NT_ASSERT(NT_SUCCESS(status)); + + + EnlpResumeThread(&enlLink->EnlThread); + + return enlQueue; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlDestroyQueue( + _In_ ENLP_QUEUE * Queue, + _In_ BOOLEAN Tx + ) +{ + LogInformation(FLAG_DRIVER, L"Queue=%p", Queue); + ENLP_PORT* port = Queue->EnlPortHandle; + + if (Tx) + { + port->TxQueueCount--; + } + else + { + port->RxQueueCount--; + } + + Queue->ArmWaitEvent = nullptr; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlRingDoorBell( + _In_ ENLP_QUEUE * Queue, + _In_ ULONG EndIndex + ) +{ + InterlockedExchange((volatile LONG *)&Queue->QueueEnd, (LONG)EndIndex); + + WdfSpinLockAcquire(Queue->Spinlock); + + if (Queue->Armed) + { + NT_ASSERT(Queue->ArmWaitEvent != NULL); + Queue->Armed = FALSE; + WdfSpinLockRelease(Queue->Spinlock); + Queue->ArmWaitEvent->Set(); + return; + } + + WdfSpinLockRelease(Queue->Spinlock); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlArmInterrupt( + _In_ ENLP_QUEUE * Queue, + _In_ BOOLEAN notificationEnabled + ) +{ + Queue->Notify = notificationEnabled; + + // TODO: InterlockedExchange reduces the throughput of the ENL from about 500MB/s to 8MB/s. -> why + //InterlockedExchange((volatile LONG *)&Queue->Notify, (LONG)notificationEnabled); +} + +_Use_decl_annotations_ +VOID +EnlIndicateQueueState( + ENLP_QUEUE * Queue, + ENL_QUEUE_STATE State + ) +{ + Queue->State = State; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlCreateLink( + _In_ ULONG ProcessorIndex, + _In_ ULONG IdealNode, + _In_ BOOLEAN Poll, + _In_ BOOLEAN InitializePacketLayout, + _Out_ ENLP_LINK ** EnlLink + ) +{ + LogInformation(FLAG_DRIVER, L"ProcessorIndex=%u", ProcessorIndex); + + auto enlLink = wil::make_unique_nothrow<ENLP_LINK>(); + RETURN_NTSTATUS_IF( + STATUS_INSUFFICIENT_RESOURCES, + ! enlLink); + + enlLink->Ts = ReadTimeStampCounter(); + enlLink->Poll = Poll; + enlLink->InitializePacketLayout = InitializePacketLayout; + + RETURN_IF_NOT_STATUS_SUCCESS( + EnlpStartThread( + ProcessorIndex, + IdealNode, + EnlpIterationRoutine, + enlLink.get(), + enlLink->Poll ? NULL : &enlLink->ArmWaitEvent, + &enlLink->EnlThread)); + + *EnlLink = enlLink.release(); + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlIsPortActive( + _In_ ENLP_LINK * EnlLink, + _In_ ULONG PortIndex + ) +{ + ENLP_PORT* port = &EnlLink->Ports[PortIndex]; + ENLP_QUEUE* txq = &port->TxQueue[0]; + NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT); + + return (txq->Queue == nullptr) ? FALSE : TRUE; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlIsLinkActive( + _In_ ENLP_LINK * EnlLink + ) +{ + return (EnlIsPortActive(EnlLink, 0) || EnlIsPortActive(EnlLink, 1)); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlActivateLinkPort( + _In_ ENLP_LINK * EnlLink, + _In_ ULONG PortIndex, + _In_ ENL_INTERRUPT_ROUTINE Interrupt, + _In_ PVOID PortContext + ) +{ + ENLP_PORT* port = &EnlLink->Ports[PortIndex]; + + LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue); + + NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, PortIndex)); + NT_FRE_ASSERT(port->TxQueueCount == 0); + NT_FRE_ASSERT(port->RxQueueCount == 0); + + port->Interrupt = Interrupt; + port->PortContext = PortContext; + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlDeactivateLinkPort( + _In_ ENLP_LINK * EnlLink, + _In_ ULONG PortIndex + ) +{ + NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT); + + ENLP_PORT* port = &EnlLink->Ports[PortIndex]; + LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue); + + NT_FRE_ASSERT(EnlIsPortActive(EnlLink, PortIndex)); + + KLockThisExclusive(EnlLink->Lock); + NT_FRE_ASSERT(!EnlpIsThreadPaused(&EnlLink->EnlThread)); + EnlpPauseThread(&EnlLink->EnlThread); + +#if _KERNEL_MODE + KeFlushQueuedDpcs(); +#endif + + port->PortContext = NULL; + + //Clears reference to only first queue - Change for all queues + port->TxQueue[0].Queue = nullptr; + port->RxQueue[0].Queue = nullptr; + + + // As long as there's one port with active queues, the EnlThread will run. + for (ULONG i = 0; i < ENLP_PORT_COUNT; i++) + { + if (EnlIsPortActive(EnlLink, i)) + { + EnlpResumeThread(&EnlLink->EnlThread); + break; + } + } +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlCloseLink( + _In_ ENLP_LINK * EnlLink + ) +{ + ULONG i; + + for (i = 0; i < ENLP_PORT_COUNT; i++) + { + NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, i)); + } + + EnlpStopThread(&EnlLink->EnlThread); + + delete EnlLink; +} + +/////////////////////////////////////////////////////////////////////////////// +// Multi link wrapper APIs // +/////////////////////////////////////////////////////////////////////////////// + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlMCreateLink( + _In_range_(1, ENL_MLINK_MAX)ULONG LinkCount, + _In_reads_(LinkCount) ULONG ProcessorIndex, + _In_ BOOLEAN Poll, + _Out_ ENL_MLINK* EnlMLink + ) +{ + NTSTATUS status = STATUS_SUCCESS; + ULONG i = 0; + + RtlZeroMemory(EnlMLink, sizeof(*EnlMLink)); + + if (LinkCount < 1 || LinkCount > ENL_MLINK_MAX || + (LinkCount & (LinkCount - 1)) != 0) + { + status = STATUS_REQUEST_NOT_ACCEPTED; + goto exit; + } + + for (i = 0; i < LinkCount; i++) + { +#ifdef _KERNEL_MODE + const BOOLEAN initializePacketLayout = TRUE; +#else + const BOOLEAN initializePacketLayout = FALSE; +#endif + status = EnlCreateLink(ProcessorIndex, 0, Poll, initializePacketLayout, &EnlMLink->LinkHandle[i]); + + if (!NT_SUCCESS(status)) + { + goto exit; + } + } + + EnlMLink->LinkCount = LinkCount; + +exit: + + if (!NT_SUCCESS(status)) + { + for (; i > 0; i--) + { + EnlCloseLink(EnlMLink->LinkHandle[i - 1]); + EnlMLink->LinkHandle[i - 1] = NULL; + } + } + RETURN_NTSTATUS_IF( + status, + status != STATUS_SUCCESS); + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void EnlSetPdoWakeSignalCallback( + _In_ ENLP_LINK * EnlLink, + _In_ EVT_ENLP_PDO_WAKE_SIGNAL* evtPdoWakeSignal, + _In_ void* Context) +{ + EnlLink->EvtWakeSignal = evtPdoWakeSignal; + EnlLink->WakeSignalContext = Context; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlArmWake( + _In_ ENLP_LINK * EnlLink +) +{ + EnlLink->ArmedForWake = TRUE; + RtlZeroMemory(&EnlLink->WakeFrame[0], sizeof(EnlLink->WakeFrame)); + EnlLink->WakeFrameSize = 0; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlDisarmWake( + _In_ ENLP_LINK * EnlLink +) +{ + EnlLink->ArmedForWake = FALSE; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +size_t +EnlCopyWakeFrame( + _In_ ENLP_LINK * EnlLink, + _Out_writes_bytes_(BufferSize) unsigned char * Buffer, + _In_ size_t BufferSize +) +{ + if (EnlLink->WakeFrameSize == 0) + { + // There was no wake + return 0; + } + + if (BufferSize < EnlLink->WakeFrameSize) + { + // Wake frame is larger than what we can indicate + return 0; + } + + RtlCopyMemory(Buffer, &EnlLink->WakeFrame[0], EnlLink->WakeFrameSize); + auto const wakeFrameSize = EnlLink->WakeFrameSize; + + // Make sure to erase the wake frame, since the network interface might have + // multiple receive queues + RtlZeroMemory(&EnlLink->WakeFrame[0], EnlLink->WakeFrameSize); + EnlLink->WakeFrameSize = 0; + + return wakeFrameSize; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlMIsPortActive( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex + ) +{ + ULONG i; + + BOOLEAN result = EnlIsPortActive(EnlMLink->LinkHandle[0], PortIndex); + + for (i = 1; i < EnlMLink->LinkCount; i++) + { + NT_FRE_ASSERT(EnlIsPortActive(EnlMLink->LinkHandle[i], PortIndex) == result); + } + + return result; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlMIsLinkActive( + _In_ CONST ENL_MLINK* EnlMLink + ) +{ + ULONG i; + + BOOLEAN result = EnlIsLinkActive(EnlMLink->LinkHandle[0]); + + for (i = 1; i < EnlMLink->LinkCount; i++) + { + NT_FRE_ASSERT(EnlIsLinkActive(EnlMLink->LinkHandle[i]) == result); + } + + return result; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlMActivateLinkPort( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex, + _In_ ENL_INTERRUPT_ROUTINE Interrupt, + _In_ PVOID PortContext + ) +{ + LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex); + + NTSTATUS status = STATUS_SUCCESS; + ULONG i; + + for (i = 0; i < EnlMLink->LinkCount; i++) + { + status = EnlActivateLinkPort( + EnlMLink->LinkHandle[i], + PortIndex, + Interrupt, + PortContext); + + if (!NT_SUCCESS(status)) + { + break; + } + } + + if (!NT_SUCCESS(status)) + { + for (; i > 0; i--) + { + EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex); + } + } + + RETURN_NTSTATUS_IF(status, + status != STATUS_SUCCESS); + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlMDeactivateLinkPort( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex + ) +{ + LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex); + + ULONG i; + + for (i = 0; i < EnlMLink->LinkCount; i++) + { + EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex); + } +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlMCloseLink( + _Inout_ ENL_MLINK* EnlMLink + ) +{ + LogInformation(FLAG_DRIVER, L"EnlMLink=%p", EnlMLink); + + ULONG i; + + for (i = 0; i < EnlMLink->LinkCount; i++) + { + EnlCloseLink(EnlMLink->LinkHandle[i]); + EnlMLink->LinkHandle[i] = NULL; + } + + EnlMLink->LinkCount = 0; +} |