diff options
| author | 5an7y-Microsoft <[email protected]> | 2026-06-19 13:23:44 -0700 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-06-19 13:23:44 -0700 |
| commit | 26741c75857c7df2fd95efe632f8f5f70b71fdbb (patch) | |
| tree | 58d74336b33f09a11d9427f8eba76f7fa84d9d47 /network/netadaptercx/netvadapterlibrary/code | |
| parent | e99ae832b48b245404f9bd750af4864247b061e8 (diff) | |
| parent | 8315a48cc6aac0c02d2218b0147457a01de4f885 (diff) | |
Merge pull request #1394 from youyanggit/WifiihvsampleDataPath
Add End-to-End NetAdapterCx/WiFiCx Sample Drivers and OEM Communication Examples
Diffstat (limited to 'network/netadaptercx/netvadapterlibrary/code')
25 files changed, 4045 insertions, 0 deletions
diff --git a/network/netadaptercx/netvadapterlibrary/code/adapter.cpp b/network/netadaptercx/netvadapterlibrary/code/adapter.cpp new file mode 100644 index 00000000..880f970d --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/adapter.cpp @@ -0,0 +1,595 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +#include "pch.hpp" + +#include <new.h> +#ifdef _KERNEL_MODE +#include <xfilter.h> +#else +#include "net/umxfilter.h" // Copied from Km XFilter.h, NETCX please move this xfilter.h into shared location +#endif +#include "netvadapter.h" +#include "rxqueue.h" +#include "txqueue.h" +#include "configuration.h" +#include "trace.h" +#include "memory.h" + +#include "adapter.tmh" + +UCHAR NetvMacAddressBase[MAC_ADDR_LEN] = { 0x22, 0x22, 0x22, 0x22, 0x00, 0x00 }; +const ULONG GSO_MAX_OFFLOAD_SIZE = 0xffff; +const ULONG GSO_MIN_SEGMENT_COUNT = 2; + +/* + * increasing beyond 1Gbps results in intermittent failure of + * netvadapter start due to buffer allocation failures in + * netadaptercx when running in nebula. + * tracked as bug 50671552 (if it doesn't get archived) + */ +static auto constexpr MAX_LINK_SPEED{1'000'000'000ull}; + +void +NetvEnlInterruptRoutine( + _Inout_ PVOID PortContext, + _In_ bool Tx +) +{ + NETPACKETQUEUE queue = (NETPACKETQUEUE)PortContext; + + if (Tx) // Tx + { + NetTxQueueNotifyMoreCompletedPacketsAvailable(queue); + } + else // Rx + { + NetRxQueueNotifyMoreReceivedPacketsAvailable(queue); + } +} + +static +EVT_PACKET_QUEUE_START + EvtTxQueueStart; + +static +EVT_PACKET_QUEUE_STOP + EvtTxQueueStop; + +static +EVT_PACKET_QUEUE_ADVANCE + EvtTxQueueAdvance; + +static +EVT_PACKET_QUEUE_CANCEL + EvtTxQueueCancel; + +static +EVT_PACKET_QUEUE_SET_NOTIFICATION_ENABLED + EvtTxQueueSetNotify; + +static +EVT_PACKET_QUEUE_START + EvtRxQueueStart; + +static +EVT_PACKET_QUEUE_STOP + EvtRxQueueStop; + +static +EVT_PACKET_QUEUE_ADVANCE + EvtRxQueueAdvance; + +static +EVT_PACKET_QUEUE_CANCEL + EvtRxQueueCancel; + +static +EVT_PACKET_QUEUE_SET_NOTIFICATION_ENABLED + EvtRxQueueSetNotify; + +NetvAdapter::NetvAdapter( + NETADAPTER Handle, + WDFDEVICE Device +) noexcept + : m_handle(Handle) + , m_device(Device) +{ +} + +NTSTATUS +NetvAdapter::Initialize( + void +) +{ + RETURN_IF_NOT_STATUS_SUCCESS( + NetvAdapterReadConfiguration(this, m_device)); + + RETURN_IF_NOT_STATUS_SUCCESS(NetvAdapterReadAddress()); + + SetLinkState(); + + NTSTATUS status = STATUS_SUCCESS; + + // Create ENL + EnlPortCreated = FALSE; + + if (NetvEnlMLink[EnlIndex].LinkCount == 0) + { + RETURN_IF_NOT_STATUS_SUCCESS( + EnlMCreateLink( + LinkCount, + LinkProcIndex, + LinkPoll, + &NetvEnlMLink[EnlIndex])); + } + + RETURN_NTSTATUS_IF(STATUS_INVALID_ADDRESS, + EnlMIsPortActive(&NetvEnlMLink[EnlIndex], EnlPortIndex)); + + status = EnlMActivateLinkPort( + &NetvEnlMLink[EnlIndex], + EnlPortIndex, + NetvEnlInterruptRoutine, + this); + + // what is the virtue of doing this? + if (NT_SUCCESS(status)) + { + EnlPortCreated = TRUE; + } + + RETURN_STATUS_SUCCESS(); +} + +void +NetvAdapter::Destroy( + void +) +{ + if (EnlPortCreated) + { + EnlMDeactivateLinkPort(&NetvEnlMLink[EnlIndex], EnlPortIndex); + EnlPortCreated = FALSE; + } + + if (EnlIndexValid && !EnlIsLinkActive(NetvEnlMLink[EnlIndex].LinkHandle[0])) + { + EnlMCloseLink(&NetvEnlMLink[EnlIndex]); + } +} + +_Use_decl_annotations_ +NTSTATUS +NetvAdapter::CreateRxQueue( + NETRXQUEUE_INIT * NetRxQueueInit + ) +{ + WDF_OBJECT_ATTRIBUTES rxAttributes; + WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&rxAttributes, NetvRxQueue); + rxAttributes.EvtDestroyCallback = [](WDFOBJECT Handle) { + NetvRxQueueGetContext(static_cast<NETPACKETQUEUE>(Handle))->Destroy(); + }; + + NET_PACKET_QUEUE_CONFIG rxConfig; + NET_PACKET_QUEUE_CONFIG_INIT( + &rxConfig, + EvtRxQueueAdvance, + EvtRxQueueSetNotify, + EvtRxQueueCancel); + rxConfig.EvtStart = EvtRxQueueStart; + rxConfig.EvtStop = EvtRxQueueStop; + + NETPACKETQUEUE rxQueue; + RETURN_IF_NOT_STATUS_SUCCESS(NetRxQueueCreate( + NetRxQueueInit, + &rxAttributes, + &rxConfig, + &rxQueue)); + + new (NetvRxQueueGetContext(rxQueue)) NetvRxQueue(rxQueue, *this); + + RETURN_STATUS_SUCCESS(); +} + +_Use_decl_annotations_ +NTSTATUS +NetvAdapter::CreateTxQueue( + NETTXQUEUE_INIT * NetTxQueueInit + ) +{ + WDF_OBJECT_ATTRIBUTES txAttributes; + WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&txAttributes, NetvTxQueue); + txAttributes.EvtDestroyCallback = [](WDFOBJECT Handle) { + NetvTxQueueGetContext(static_cast<NETPACKETQUEUE>(Handle))->Destroy(); + }; + + NET_PACKET_QUEUE_CONFIG txConfig; + NET_PACKET_QUEUE_CONFIG_INIT( + &txConfig, + EvtTxQueueAdvance, + EvtTxQueueSetNotify, + EvtTxQueueCancel); + txConfig.EvtStart = EvtTxQueueStart; + txConfig.EvtStop = EvtTxQueueStop; + + NETPACKETQUEUE txQueue; + RETURN_IF_NOT_STATUS_SUCCESS(NetTxQueueCreate( + NetTxQueueInit, + &txAttributes, + &txConfig, + &txQueue)); + + new (NetvTxQueueGetContext(txQueue)) NetvTxQueue(txQueue, *this); + + RETURN_STATUS_SUCCESS(); +} + +void NetvAdapter::SetPdoWakeSignalCallback(_In_ EVT_PDO_WAKE_SIGNAL* evtPdoWakeSignal, _In_ void* context) +{ + EnlSetPdoWakeSignalCallback(NetvEnlMLink[EnlIndex].LinkHandle[0], evtPdoWakeSignal, context); +} + +_Use_decl_annotations_ +_IRQL_requires_max_(PASSIVE_LEVEL) +void NetvAdapter::ArmWakeFromS0(void) +{ + if (EnlPortCreated) + { + ENLP_LINK* enLinkHandle = NetvEnlMLink[EnlIndex].LinkHandle[0]; + EnlArmWake(enLinkHandle); + } +} + +_Use_decl_annotations_ +_IRQL_requires_max_(PASSIVE_LEVEL) +void NetvAdapter::DisarmWakeFromS0(void) +{ + if (EnlPortCreated) + { + ENLP_LINK* enLinkHandle = NetvEnlMLink[EnlIndex].LinkHandle[0]; + EnlDisarmWake(enLinkHandle); + } +} + +_Use_decl_annotations_ +void +NetvAdapter::SetLinkState( + void +) const +{ + NET_ADAPTER_AUTO_NEGOTIATION_FLAGS autoNegotiationFlags{NetAdapterAutoNegotiationFlagNone}; + if (LinkAutoNeg) + { + autoNegotiationFlags |= + NetAdapterAutoNegotiationFlagXmitLinkSpeedAutoNegotiated | + NetAdapterAutoNegotiationFlagRcvLinkSpeedautoNegotiated | + NetAdapterAutoNegotiationFlagDuplexAutoNegotiated; + } + if (FlowControl != NetvFlowControlDisabled) + { + autoNegotiationFlags |= + NetAdapterAutoNegotiationFlagPauseFunctionsAutoNegotiated; + } + + NET_ADAPTER_PAUSE_FUNCTION_TYPE pauseFunctions{NetAdapterPauseFunctionTypeUnknown}; + switch (FlowControl) + { + case NetvFlowControlDisabled: + pauseFunctions = NetAdapterPauseFunctionTypeUnsupported; + break; + case NetvFlowControlRxEnabled: + pauseFunctions = NetAdapterPauseFunctionTypeReceiveOnly; + break; + case NetvFlowControlTxEnabled: + pauseFunctions = NetAdapterPauseFunctionTypeSendOnly; + break; + case NetvFlowControlTxRxEnabled: + pauseFunctions = NetAdapterPauseFunctionTypeSendAndReceive; + break; + } + + NET_ADAPTER_LINK_STATE linkState; + NET_ADAPTER_LINK_STATE_INIT( + &linkState, + MAX_LINK_SPEED, + MediaConnectStateConnected, + MediaDuplexStateFull, + pauseFunctions, + autoNegotiationFlags); + NetAdapterSetLinkState(m_handle, &linkState); +} + +static +void +EvtSetReceiveFilter( + _In_ NETADAPTER NetAdapter, + _In_ NETRECEIVEFILTER Handle + ) +{ + NetvAdapter* adapter = NetvAdapterGetContextFromWDFObject(NetAdapter); + + adapter->PacketFilter = NetReceiveFilterGetPacketFilter(Handle); + + adapter->NumMulticastAddresses = (ULONG)NetReceiveFilterGetMulticastAddressCount(Handle); + + RtlZeroMemory(adapter->MulticastAddressList, + sizeof(NET_ADAPTER_LINK_LAYER_ADDRESS) * MAX_MULTICAST_LIST_SIZE); + + if (adapter->NumMulticastAddresses != 0U) + { + NET_ADAPTER_LINK_LAYER_ADDRESS const * MulticastAddressList = NetReceiveFilterGetMulticastAddressList(Handle); + RtlCopyMemory(adapter->MulticastAddressList, + MulticastAddressList, + sizeof(NET_ADAPTER_LINK_LAYER_ADDRESS) * adapter->NumMulticastAddresses); + } +} + +static +_IRQL_requires_same_ +_IRQL_requires_max_(PASSIVE_LEVEL) +void +NTAPI +EvtNetAdapterOffloadSetRxXSum( + _In_ NETADAPTER Adapter, + _In_ NETOFFLOAD Offload + ) +{ + UNREFERENCED_PARAMETER((Adapter, Offload)); + ASSERT(NetOffloadIsRxChecksumIPv4Enabled(Offload)); + ASSERT(NetOffloadIsRxChecksumUdpEnabled(Offload)); +} + +static +_IRQL_requires_same_ +_IRQL_requires_max_(PASSIVE_LEVEL) +void +NTAPI +EvtNetAdapterOffloadSetGso( + _In_ NETADAPTER Adapter, + _In_ NETOFFLOAD Offload + ) +{ + auto adapter = NetvAdapterGetContextFromWDFObject(Adapter); + if (adapter->UsoEnabled) + { + // + // Since netvadapter only converts USO sends into URO receives, + // both must be enabled together, or disabled together. + // The order of callbacks is nondeterministic, so this can't assert + // that URO is enabled, since it might not have been enabled yet. + // This assert is so anyone using netvadapter knows that USO has + // been disabled after being enabled. + // + NT_FRE_ASSERTMSG("USO can't be disabled after being enabled", NetOffloadIsUsoIPv4Enabled(Offload)); + NT_FRE_ASSERTMSG("USO can't be disabled after being enabled", NetOffloadIsUsoIPv6Enabled(Offload)); + } + else + { + adapter->UsoEnabled = NetOffloadIsUsoIPv4Enabled(Offload) && NetOffloadIsUsoIPv6Enabled(Offload); + } +} + +static +_IRQL_requires_same_ +_IRQL_requires_max_(PASSIVE_LEVEL) +void +NTAPI +EvtNetAdapterOffloadSetRsc( + _In_ NETADAPTER Adapter, + _In_ NETOFFLOAD Offload + ) +{ + auto adapter = NetvAdapterGetContextFromWDFObject(Adapter); + if (adapter->UroEnabled) + { + // + // Since netvadapter only converts USO sends into URO receives, + // both must be enabled together, or disabled together. + // The order of callbacks is nondeterministic, so this can't assert + // that USO is enabled, since it might not have been enabled yet. + // This assert is so anyone using netvadapter knows that URO has + // been disabled after being enabled. + // This assert is important because if URO is disabled, netvadapter + // will forward the USO send without fixing it up or segmenting it, + // and the stack may behave badly. + // + NT_FRE_ASSERTMSG("URO can't be disabled after being enabled", NetOffloadIsUdpRscEnabled(Offload)); + } + else + { + adapter->UroEnabled = NetOffloadIsUdpRscEnabled(Offload); + } +} + +static +void +NetvAdapterSetUsoUroOffloadCapabilities( + _In_ NETADAPTER Adapter + ) +{ + NET_ADAPTER_OFFLOAD_GSO_CAPABILITIES gsoCapabilities; + NET_ADAPTER_OFFLOAD_GSO_CAPABILITIES_INIT( + &gsoCapabilities, + NetAdapterOffloadLayer3FlagIPv4NoOptions | NetAdapterOffloadLayer3FlagIPv6NoExtensions, + NetAdapterOffloadLayer4FlagUdp, + GSO_MAX_OFFLOAD_SIZE, + GSO_MIN_SEGMENT_COUNT, + EvtNetAdapterOffloadSetGso); + + NET_ADAPTER_OFFLOAD_RSC_CAPABILITIES rscCapabilities; + NET_ADAPTER_OFFLOAD_RSC_CAPABILITIES_INIT( + &rscCapabilities, + NetAdapterOffloadLayer3FlagIPv4NoOptions | NetAdapterOffloadLayer3FlagIPv6NoExtensions, + NetAdapterOffloadLayer4FlagUdp, + EvtNetAdapterOffloadSetRsc); + rscCapabilities.TcpTimestampOption = FALSE; + + NetAdapterOffloadSetGsoCapabilities(Adapter, &gsoCapabilities); + NetAdapterOffloadSetRscCapabilities(Adapter, &rscCapabilities); +} + +_Use_decl_annotations_ +NTSTATUS NetvAdapter::ConfigureDataCapabilities() +{ + + + NET_ADAPTER_TX_CAPABILITIES txCapabilities; + NET_ADAPTER_TX_CAPABILITIES_INIT(&txCapabilities, MAX_TX_QUEUES); + + NET_ADAPTER_RX_CAPABILITIES rxCapabilities; + NET_ADAPTER_RX_CAPABILITIES_INIT_SYSTEM_MANAGED(&rxCapabilities, MAX_RX_BUFFER_SIZE, MAX_RX_QUEUES); + + NET_ADAPTER_LINK_LAYER_CAPABILITIES linkLayerCapabilities; + NET_ADAPTER_LINK_LAYER_CAPABILITIES_INIT(&linkLayerCapabilities, MAX_LINK_SPEED, MAX_LINK_SPEED); + + NET_ADAPTER_RECEIVE_FILTER_CAPABILITIES receiveFilterCapabilities; + NET_ADAPTER_RECEIVE_FILTER_CAPABILITIES_INIT(&receiveFilterCapabilities, EvtSetReceiveFilter); + receiveFilterCapabilities.SupportedPacketFilters = NETV_SUPPORTED_FILTERS; + receiveFilterCapabilities.MaximumMulticastAddresses = MAX_MULTICAST_LIST_SIZE; + + NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES xsumCapabilities; + NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES_INIT(&xsumCapabilities, EvtNetAdapterOffloadSetRxXSum); + + NetAdapterSetLinkLayerCapabilities(m_handle, &linkLayerCapabilities); + NetAdapterSetLinkLayerMtuSize(m_handle, MTU_SIZE); + NetAdapterSetDataPathCapabilities(m_handle, &txCapabilities, &rxCapabilities); + NetAdapterSetReceiveFilterCapabilities(m_handle, &receiveFilterCapabilities); + NetAdapterOffloadSetRxChecksumCapabilities(m_handle, &xsumCapabilities); + + NET_ADAPTER_LINK_LAYER_ADDRESS netvLinkAddress; + NET_ADAPTER_LINK_LAYER_ADDRESS_INIT(&netvLinkAddress, MAC_ADDR_LEN, (CONST UCHAR*)&(PermanentAddress.Address)); + + NetAdapterSetPermanentLinkLayerAddress(m_handle, &netvLinkAddress); + NetAdapterSetCurrentLinkLayerAddress(m_handle, &netvLinkAddress); + + if (EnableUsoUro) + { + NetvAdapterSetUsoUroOffloadCapabilities(m_handle); + } + + RETURN_STATUS_SUCCESS(); +} + +NTSTATUS +NetvAdapter::NetvAdapterReadAddress() +{ + PermanentAddress.Length = MAC_ADDR_LEN; + + RETURN_NTSTATUS_IF(STATUS_INVALID_ADDRESS, + MACLastByte < 1 || + MACLastByte > MAX_ADAPTER_COUNT); + + ETH_COPY_NETWORK_ADDRESS(PermanentAddress.Address, NetvMacAddressBase); + PermanentAddress.Address[MAC_ADDR_LEN - 1] = (UCHAR) MACLastByte; + + if (ETH_IS_MULTICAST(PermanentAddress.Address) || + ETH_IS_BROADCAST(PermanentAddress.Address)) + { + RETURN_IF_NOT_STATUS_SUCCESS(STATUS_INVALID_ADDRESS); + } + + RtlCopyMemory( + &CurrentAddress, + &PermanentAddress, + sizeof(PermanentAddress) + ); + + EnlIndex = (MACLastByte - 1) >> 1; + EnlPortIndex = (MACLastByte - 1) & 1; + EnlIndexValid = TRUE; + + RETURN_STATUS_SUCCESS(); +} + +_Use_decl_annotations_ +void +EvtTxQueueStart( + NETPACKETQUEUE Queue +) +{ + NetvTxQueueGetContext(Queue)->Start(); +} + +_Use_decl_annotations_ +void +EvtTxQueueStop( + NETPACKETQUEUE Queue +) +{ + NetvTxQueueGetContext(Queue)->Stop(); +} + +_Use_decl_annotations_ +void +EvtTxQueueAdvance( + NETPACKETQUEUE Queue +) +{ + NetvTxQueueGetContext(Queue)->Advance(); +} + +_Use_decl_annotations_ +void +EvtTxQueueCancel( + NETPACKETQUEUE Queue +) +{ + NetvTxQueueGetContext(Queue)->Cancel(); +} + +_Use_decl_annotations_ +void +EvtTxQueueSetNotify( + NETPACKETQUEUE Queue, + BOOLEAN Enable +) +{ + NetvTxQueueGetContext(Queue)->SetNotify(Enable); +} + +_Use_decl_annotations_ +void +EvtRxQueueStart( + NETPACKETQUEUE Queue +) +{ + NetvRxQueueGetContext(Queue)->Start(); +} + +_Use_decl_annotations_ +void +EvtRxQueueStop( + NETPACKETQUEUE Queue +) +{ + NetvRxQueueGetContext(Queue)->Stop(); +} + + +_Use_decl_annotations_ +void +EvtRxQueueAdvance( + NETPACKETQUEUE Queue +) +{ + NetvRxQueueGetContext(Queue)->Advance(); +} + +_Use_decl_annotations_ +void +EvtRxQueueCancel( + NETPACKETQUEUE Queue +) +{ + NetvRxQueueGetContext(Queue)->Cancel(); +} + +_Use_decl_annotations_ +void +EvtRxQueueSetNotify( + NETPACKETQUEUE Queue, + BOOLEAN Enable +) +{ + NetvRxQueueGetContext(Queue)->SetNotify(Enable); +} diff --git a/network/netadaptercx/netvadapterlibrary/code/configuration.cpp b/network/netadaptercx/netvadapterlibrary/code/configuration.cpp new file mode 100644 index 00000000..7482ef84 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/configuration.cpp @@ -0,0 +1,102 @@ +#include "pch.hpp" +#include "netvadapter.h" + +#include "trace.h" +#include "configuration.tmh" + +typedef struct _NETVADAPTER_ADVANCED_PROPERTY +{ + UNICODE_STRING RegName; // variable name text + UINT32 FieldOffset; // offset to NetvAdapter field + UINT32 FieldSize; // size (in bytes) of the field + UINT32 Default; // default value to use + UINT32 Min; // minimum value allowed + UINT32 Max; // maximum value allowed +} NETVADAPTER_ADVANCED_PROPERTY; + +#define NETV_OFFSET(field) ((UINT32)FIELD_OFFSET(NetvAdapter,field)) +#define NETV_SIZE(field) RTL_FIELD_SIZE(NetvAdapter,field) + +#define CONSTANT_UNICODE_STRING(s) {sizeof( s ) - sizeof( WCHAR ), sizeof( s ), s } + +NETVADAPTER_ADVANCED_PROPERTY NetvSupportedProperties[] = +{ + // reg value name - Offset in NetvAdapter - Field size - Default Value - Min - Max + + // Standard Keywords + { CONSTANT_UNICODE_STRING(L"MACLastByte"), NETV_OFFSET(MACLastByte), NETV_SIZE(MACLastByte), 0, 0, 254 }, + { CONSTANT_UNICODE_STRING(L"LinkProcIndex"), NETV_OFFSET(LinkProcIndex), NETV_SIZE(LinkProcIndex), 1000, 0, 1023 }, + { CONSTANT_UNICODE_STRING(L"S0Idle"), NETV_OFFSET(S0Idle), NETV_SIZE(S0Idle), 0, 0, 1 }, + { CONSTANT_UNICODE_STRING(L"EnableUsoUro"), NETV_OFFSET(EnableUsoUro), NETV_SIZE(EnableUsoUro), 0, 0, 1 }, +}; + +NTSTATUS +NetvAdapterReadConfiguration( + NetvAdapter *Adapter, + WDFDEVICE Device + ) +{ + NTSTATUS status = STATUS_SUCCESS; + + NETCONFIGURATION configuration; + RETURN_IF_NOT_STATUS_SUCCESS( + NetDeviceOpenConfiguration(Device, WDF_NO_OBJECT_ATTRIBUTES, &configuration)); + + // read all the registry values + for (auto &property : NetvSupportedProperties) + { + // Driver should NOT fail the initialization only because it can not + // read the registry + auto pointer = (PUCHAR)Adapter + property.FieldOffset; + + // Get the configuration value for a specific parameter. Under NT the + // parameters are all read in as DWORDs. + ULONG value = 0; + + status = NetConfigurationQueryUlong( + configuration, + NET_CONFIGURATION_QUERY_ULONG_NO_FLAGS, + &property.RegName, + &value); + + // Store the value in the adapter structure. + switch (property.FieldSize) + { + case 1: + *((PUCHAR)pointer) = (UCHAR)value; + break; + + case 2: + *((PUSHORT)pointer) = (USHORT)value; + break; + + case 4: + *((PULONG)pointer) = (ULONG)value; + break; + + default: + break; + } + + // If the parameter was present, then check its value for validity. + if (NT_SUCCESS(status)) + { + // Check that param value is not too small or too large + + if (value < property.Min || + value > property.Max) + { + value = property.Default; + } + } + else + { + value = property.Default; + status = STATUS_SUCCESS; + } + } + + NetConfigurationClose(configuration); + + RETURN_STATUS_SUCCESS(); +} diff --git a/network/netadaptercx/netvadapterlibrary/code/configuration.h b/network/netadaptercx/netvadapterlibrary/code/configuration.h new file mode 100644 index 00000000..0bfe0170 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/configuration.h @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +NTSTATUS +NetvAdapterReadConfiguration( + NetvAdapter *Adapter, + WDFDEVICE Device + ); + diff --git a/network/netadaptercx/netvadapterlibrary/code/enl.cpp b/network/netadaptercx/netvadapterlibrary/code/enl.cpp new file mode 100644 index 00000000..709c5a1e --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/enl.cpp @@ -0,0 +1,1001 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#include "pch.hpp" +#include <stdlib.h> +#include "netvadapter.h" +#include "rxqueue.h" +#include "txqueue.h" +#include "trace.h" +#include "memory.h" +#include "rtl\KLockHolder.h" +#include "enl.tmh" + +/////////////////////////////////////////////////////////////////////////////// +// BEGIN Generic execution engine thread lib // +/////////////////////////////////////////////////////////////////////////////// + +ENL_MLINK NetvEnlMLink[MAX_ADAPTER_COUNT / 2]; + +/*++ +The iteration routine performs one full pass over all input queues and +any internal queues (that might be holding previous items waiting to be +processed). +--*/ + +static +SIZE_T +CopyTxPacketDataToBuffer( + _Out_writes_bytes_(BufferLength) PUCHAR BufferDest, + _In_ NET_RING_PACKET_ITERATOR const * Iterator, + _In_ NET_EXTENSION const * VirtualAddressExtension, + _In_ SIZE_T BufferLength) +{ + SIZE_T bytesCopied = 0; + + for (NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(Iterator); + NetFragmentIteratorHasAny(&fi) && (BufferLength > 0); + NetFragmentIteratorAdvance(&fi)) + { + NET_FRAGMENT const * fragment = NetFragmentIteratorGetFragment(&fi); + NET_FRAGMENT_VIRTUAL_ADDRESS const * virtualAddress = + NetExtensionGetFragmentVirtualAddress(VirtualAddressExtension, NetFragmentIteratorGetIndex(&fi)); + + UCHAR const * pPacketData = (UCHAR const *)virtualAddress->VirtualAddress + fragment->Offset; + SIZE_T bytesToCopy = + (BufferLength < (SIZE_T) fragment->ValidLength) ? BufferLength : (SIZE_T) fragment->ValidLength; + RtlCopyMemory(BufferDest, pPacketData, bytesToCopy); + + bytesCopied += bytesToCopy; + BufferDest += bytesToCopy; + BufferLength -= bytesToCopy; + } + + return bytesCopied; +} + +VOID +EnlpAffinitizeThread( + _In_ ULONG ProcIndex, + _In_ ULONG IdealNode + ) +{ + LogInformation(FLAG_DRIVER, L"ProcIndex=%u", ProcIndex); + NTSTATUS status; + PROCESSOR_NUMBER procNumber = { 0 }; + GROUP_AFFINITY affinity = { 0 }; + + if (ProcIndex == 999) + { + // No affinity at all + return; + } + + if (ProcIndex == 1000 || (ProcIndex >= 1001 && ProcIndex <= 1004)) + { + KeQueryNodeActiveAffinity((USHORT)IdealNode, &affinity, NULL); + + if (ProcIndex == 1000) + { + // Affinitize to all the procs in the Node + NOTHING; + } + else + { + // Affinitize to the highest numbered proc in the Node for 1001, + // next highest numbered proc for 1002, ... + ULONG index; +#ifdef _WIN64 + NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse64(&index, (ULONG64)affinity.Mask)); + affinity.Mask = (KAFFINITY)((ULONG64)1 << (index - (ProcIndex - 1001))); +#else + NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse(&index, (ULONG)affinity.Mask)); + affinity.Mask = (KAFFINITY)((ULONG)1 << (index - (ProcIndex - 1001))); +#endif + } + } + else + { + // Affinitize the specifically requested proc + status = KeGetProcessorNumberFromIndex(ProcIndex, &procNumber); + NT_FRE_ASSERTMSG("Bad ProcIndex", NT_SUCCESS(status)); + + affinity.Group = procNumber.Group; + affinity.Mask = AFFINITY_MASK(procNumber.Number); + } + + EnlThreadSetAffinity(&affinity, NULL); +} + +ENL_START_ROUTINE EnlpThreadRoutine; + +NTSTATUS +EnlpStartThread( + _In_ ULONG ProcIndex, + _In_ ULONG IdealNode, + _In_ ENLP_ITERATION_ROUTINE* IterationRoutine, + _In_ PVOID IterationContext, + _In_opt_ KAutoEvent* ArmWaitEvent, + _Out_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"ProcIndex=%u EnlThread=%p", ProcIndex, EnlThread); + + EnlThread->PauseRequested = TRUE; + EnlThread->StopRequested = FALSE; + EnlThread->IterationRoutine = IterationRoutine; + EnlThread->IterationContext = IterationContext; + EnlThread->ArmWaitEvent = ArmWaitEvent; + + EnlThread->ProcIndex = ProcIndex; + EnlThread->IdealNode = IdealNode; + + RETURN_IF_NOT_STATUS_SUCCESS( + EnlThreadCreate(EnlpThreadRoutine, EnlThread, EnlThread->Thread)); + + EnlThreadSetPriority(EnlThread->Thread, 15); + + RETURN_STATUS_SUCCESS(); +} + +BOOLEAN +EnlpIsThreadPaused( + _In_ CONST ENLP_THREAD_STATE* EnlThread + ) +{ + return EnlThread->PauseRequested; +} + +VOID +EnlpPauseThread( + _Inout_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread); + if (!EnlpIsThreadPaused(EnlThread)) + { + WriteBooleanNoFence(&EnlThread->PauseRequested, TRUE); + if (EnlThread->ArmWaitEvent != NULL) + { + EnlThread->ArmWaitEvent->Set(); + } + EnlThread->PausingEvent.Wait(); + } +} + +VOID +EnlpResumeThread( + _Inout_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread); + if (EnlpIsThreadPaused(EnlThread)) + { + WriteBooleanNoFence(&EnlThread->PauseRequested, FALSE); + KeMemoryBarrier(); + EnlThread->ResumeEvent.Set(); + } +} + +VOID +EnlpStopThread( + _Inout_ ENLP_THREAD_STATE* EnlThread + ) +{ + LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread); + if (EnlThread->Thread != NULL) + { + WriteBooleanNoFence(&EnlThread->StopRequested, TRUE); + EnlpResumeThread(EnlThread); // in case thread was paused + EnlThreadWaitForTermination(EnlThread->Thread); + EnlThread->Thread.reset(); + } +} + +ENL_THREAD_ROUTINE_RETURN +EnlpThreadRoutine( + _In_ PVOID Context + ) +{ + ENLP_THREAD_STATE* enlThread = (ENLP_THREAD_STATE*)Context; + + // + // Affinitize the thread + // + EnlpAffinitizeThread(enlThread->ProcIndex, enlThread->IdealNode); + + // + // ENL thread starts at paused state. + // + enlThread->ResumeEvent.Wait(); + + for (;;) + { + + if (ReadBooleanNoFence(&enlThread->PauseRequested)) + { + enlThread->PausingEvent.Set(); + enlThread->ResumeEvent.Wait(); + } + + if (ReadBooleanNoFence(&enlThread->StopRequested)) + { + break; + } + + enlThread->IterationRoutine(enlThread->IterationContext); + } + + return ENL_THREAD_ROUTINE_RETURN(); +} + +/////////////////////////////////////////////////////////////////////////////// +// END Generic execution engine thread lib // +/////////////////////////////////////////////////////////////////////////////// + +BOOLEAN +enlpCheckAndArmQueue( + _Inout_ ENLP_QUEUE* Q + ) +{ + BOOLEAN armed = FALSE; + auto ringBuffer = Q->Queue->GetPacketRing(); + // If Q is still empty, take the lock, and if still empty under lock, then + // arm it. + if (ringBuffer->BeginIndex == ringBuffer->EndIndex) + { + WdfSpinLockAcquire(Q->Spinlock); + + if (ringBuffer->BeginIndex == ringBuffer->EndIndex) + { + armed = TRUE; + Q->Armed = TRUE; + } + + WdfSpinLockRelease(Q->Spinlock); + } + + return armed; +} + +VOID +enlpArmAndWait( + _Inout_ ENLP_LINK* EnlLink + ) +{ + ULONG pi, ci; + + for (pi = 0; pi < ENLP_PORT_COUNT; pi++) + { + ENLP_PORT* port = &EnlLink->Ports[pi]; + + for (ci = 0; ci < port->TxQueueCount; ci++) + { + if (!enlpCheckAndArmQueue(port->TxQueue)) + { + return; + } + } + } + + EnlLink->ArmWaitEvent.Wait(); +} + +VOID +EnlpIterationRoutine( + _In_ PVOID IterationContext + ) +{ + auto enlLink = reinterpret_cast<ENLP_LINK *>(IterationContext); + bool emptyTx = true; + // + // Drain up to TX_BATCH_COUNT items from all tx queues first. + // + + for (size_t pi = 0U; pi < ENLP_PORT_COUNT; pi++) + { + auto txport = &enlLink->Ports[pi]; + auto rxport = &enlLink->Ports[pi ^ 0x1]; // 0 -> 1, 1 -> 0 + + for (size_t ci = 0U; ci < txport->TxQueueCount; ci++) + { + auto txq = &txport->TxQueue[ci]; + if (txq->State != Started) + continue; + + NET_RING_PACKET_ITERATOR txPi = { + txq->Queue->m_rings, nullptr, txq->QueueNext, txq->QueueEnd + }; + + while (NetPacketIteratorHasAny(&txPi)) + { + bool rxDrop = TRUE; + auto txPacket = NetPacketIteratorGetPacket(&txPi); + + emptyTx = FALSE; + + if (enlLink->ArmedForWake) + { + // Save wake packet and trigger a wake signal, any other frames queued after the wake + // packet will be dropped + enlLink->WakeFrameSize = CopyTxPacketDataToBuffer( + &enlLink->WakeFrame[0], + &txPi, + &txq->TxQueue->VirtualAddressExtension, + sizeof(enlLink->WakeFrame)); +#if _KERNEL_MODE + enlLink->EvtWakeSignal(enlLink->WakeSignalContext); +#endif + + // Make sure to disarm wake, otherwise this thread might overwrite the original wake frame + EnlDisarmWake(enlLink); + } + + if (rxport->RxQueueCount > 0) + { + //TODO: Currently does 1:1 mapping between Tx and Rx. Need to set up indirection table + auto rxq = &rxport->RxQueue[ci]; + + if (rxq->State == Started) + { + NET_RING_FRAGMENT_ITERATOR rxFi = { + rxq->Queue->m_rings, nullptr, rxq->QueueNext, rxq->QueueEnd + }; + + auto rxPi = NetRingGetPostPackets(rxq->Queue->m_rings); + + if (NetFragmentIteratorHasAny(&rxFi) && NetPacketIteratorHasAny(&rxPi)) + { + rxDrop = FALSE; + + auto fragment = NetFragmentIteratorGetFragment(&rxFi); + BYTE* fragmentBuffer = nullptr; + + auto const rxVirtualAddress = + NetExtensionGetFragmentVirtualAddress( + &rxq->RxQueue->VirtualAddressExtension, + NetFragmentIteratorGetIndex(&rxFi)); + + fragmentBuffer = static_cast<BYTE *>(rxVirtualAddress->VirtualAddress); + + fragment->Offset = 0; + fragment->ValidLength = + CopyTxPacketDataToBuffer( + fragmentBuffer, + &txPi, + &txq->TxQueue->VirtualAddressExtension, + static_cast<SIZE_T>(fragment->Capacity)); + + auto rxPacket = NetPacketIteratorGetPacket(&rxPi); + rxPacket->FragmentIndex = NetFragmentIteratorGetIndex(&rxFi); + rxPacket->FragmentCount = 1; + + if (enlLink->InitializePacketLayout) + { + rxPacket->Layout = txPacket->Layout; + } + else + { + rxPacket->Layout = {}; + } + + if (rxq->RxQueue->RxXSumExtension.Enabled) + { + auto RxXSum = NetExtensionGetPacketChecksum( + &rxq->RxQueue->RxXSumExtension, + NetPacketIteratorGetIndex(&rxPi)); + RxXSum->Layer2 = NetPacketRxChecksumEvaluationNotChecked; + RxXSum->Layer3 = NetPacketRxChecksumEvaluationValid; + RxXSum->Layer4 = NetPacketRxChecksumEvaluationValid; + } + + if (rxq->RxQueue->UdpRscExtension.Enabled && + txq->TxQueue->UsoExtension.Enabled) + { + auto txUso = NetExtensionGetPacketGso( + &txq->TxQueue->UsoExtension, + NetPacketIteratorGetIndex(&txPi)); + if (txPacket->Layout.Layer4Type == NetPacketLayer4TypeUdp && + txUso->UDP.Mss > 0) + { + if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv6NoExtensions) + { + UINT16* ipv6PayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 4); + *ipv6PayloadLength = _byteswap_ushort( + (USHORT)(fragment->ValidLength - + (txPacket->Layout.Layer2HeaderLength + txPacket->Layout.Layer3HeaderLength))); + } + else if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv4NoOptions) + { + UINT16* ipv4TotalLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 2); + *ipv4TotalLength = _byteswap_ushort((USHORT)(fragment->ValidLength - txPacket->Layout.Layer2HeaderLength )); + NT_FRE_ASSERT(*ipv4TotalLength > 0); + } + + UINT16* udpPayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + + txPacket->Layout.Layer3HeaderLength + 4); + *udpPayloadLength = _byteswap_ushort( + (USHORT)(fragment->ValidLength - + (txPacket->Layout.Layer3HeaderLength + txPacket->Layout.Layer2HeaderLength))); + + // Set the checksum to zero + UINT16* udpChecksum = udpPayloadLength + 1; + *udpChecksum = 0; + + auto rxUro = NetExtensionGetPacketRsc( + &rxq->RxQueue->UdpRscExtension, + NetPacketIteratorGetIndex(&rxPi)); + rxUro->UDP.CoalescedSegmentSize = (UINT16)txUso->UDP.Mss; + rxUro->UDP.CoalescedSegmentCount = + (UINT16)((fragment->ValidLength - txPacket->Layout.Layer3HeaderLength - txPacket->Layout.Layer4HeaderLength + + txUso->UDP.Mss - 1) / txUso->UDP.Mss); + NT_FRE_ASSERT(rxUro->UDP.CoalescedSegmentCount > 1); + NT_FRE_ASSERT(*udpPayloadLength >= 8); + } + } + + // prevent any reordering the tx/rx completion flag + KeMemoryBarrier(); + + // Use Scratch field as completion flag for the rx fragment + fragment->Scratch = 1; + rxPacket->Scratch = 1; + + NetFragmentIteratorAdvance(&rxFi); + NetPacketIteratorAdvance(&rxPi); + + rxq->QueueNext = NetFragmentIteratorGetIndex(&rxFi); + NetPacketIteratorSet(&rxPi); + + if (rxq->Notify) + { + rxport->Interrupt(rxq->Queue->m_handle, rxq->TxRx); + } + } + } + } + + if (rxDrop) + { + // TODO - add rxdrop stat + } + + // Use Scratch field as completion flag for the tx packet + txPacket->Scratch = 1; + NetPacketIteratorAdvance(&txPi); + } + + if (!emptyTx) + { + if (txq->Notify) + { + txport->Interrupt(txq->Queue->m_handle, txq->TxRx); + } + + // Store the next index so that the EnlThread knows which packet to start from in the next iteration + txq->QueueNext = NetPacketIteratorGetIndex(&txPi); + } + } + } + + ULONG64 ts; + + if (emptyTx) + { + if (enlLink->Poll == FALSE) + { + enlpArmAndWait(enlLink); + } + + ts = ReadTimeStampCounter(); + enlLink->EmptyTicks += (ts - enlLink->Ts); + } + else + { + ts = ReadTimeStampCounter(); + enlLink->BusyTicks += (ts - enlLink->Ts); + } + + enlLink->Ts = ts; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +ENLP_QUEUE * +EnlCreateQueue( + _In_ NETPACKETQUEUE Queue, + _In_ BOOLEAN Tx + ) +{ + ENLP_LINK* enlLink; + ENLP_PORT* port; + ENLP_QUEUE* enlQueue; + NTSTATUS status; + + if (Tx) // Tx + { + NetvTxQueue* netvTxQueue = NetvTxQueueGetContext(Queue); + enlLink = NetvEnlMLink[netvTxQueue->m_adapter.EnlIndex].LinkHandle[0]; + port = &enlLink->Ports[netvTxQueue->m_adapter.EnlPortIndex]; + enlQueue = &port->TxQueue[0]; // Change to accommodate multiple Queues + enlQueue->Queue = netvTxQueue; + enlQueue->TxQueue = netvTxQueue; + enlQueue->State = Stopped; + enlQueue->ArmWaitEvent = &enlLink->ArmWaitEvent; + enlQueue->QueueNext = 0; + enlQueue->QueueEnd = 0; + enlQueue->TxRx = TX; + enlQueue->EnlPortHandle = port; + port->TxQueueCount++; + LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu TxQueue=%p", + &netvTxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvTxQueue); + } + else // Rx + { + NetvRxQueue* netvRxQueue = NetvRxQueueGetContext(Queue); + enlLink = NetvEnlMLink[netvRxQueue->m_adapter.EnlIndex].LinkHandle[0]; + port = &enlLink->Ports[netvRxQueue->m_adapter.EnlPortIndex]; + enlQueue = &port->RxQueue[0]; // Change to accommodate multiple Queues + enlQueue->Queue = netvRxQueue; + enlQueue->RxQueue = netvRxQueue; + enlQueue->State = Stopped; + enlQueue->QueueNext = 0; + enlQueue->QueueEnd = 0; + enlQueue->TxRx = RX; + enlQueue->EnlPortHandle = port; + port->RxQueueCount++; + LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu RxQueue=%p", + &netvRxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvRxQueue); + } + + // Create WDF spinlock for the queue, parent to the queue's WDF handle + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT(&attributes); + attributes.ParentObject = enlQueue->Queue->m_handle; + status = WdfSpinLockCreate(&attributes, &enlQueue->Spinlock); + NT_ASSERT(NT_SUCCESS(status)); + + + EnlpResumeThread(&enlLink->EnlThread); + + return enlQueue; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlDestroyQueue( + _In_ ENLP_QUEUE * Queue, + _In_ BOOLEAN Tx + ) +{ + LogInformation(FLAG_DRIVER, L"Queue=%p", Queue); + ENLP_PORT* port = Queue->EnlPortHandle; + + if (Tx) + { + port->TxQueueCount--; + } + else + { + port->RxQueueCount--; + } + + Queue->ArmWaitEvent = nullptr; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlRingDoorBell( + _In_ ENLP_QUEUE * Queue, + _In_ ULONG EndIndex + ) +{ + InterlockedExchange((volatile LONG *)&Queue->QueueEnd, (LONG)EndIndex); + + WdfSpinLockAcquire(Queue->Spinlock); + + if (Queue->Armed) + { + NT_ASSERT(Queue->ArmWaitEvent != NULL); + Queue->Armed = FALSE; + WdfSpinLockRelease(Queue->Spinlock); + Queue->ArmWaitEvent->Set(); + return; + } + + WdfSpinLockRelease(Queue->Spinlock); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlArmInterrupt( + _In_ ENLP_QUEUE * Queue, + _In_ BOOLEAN notificationEnabled + ) +{ + Queue->Notify = notificationEnabled; + + // TODO: InterlockedExchange reduces the throughput of the ENL from about 500MB/s to 8MB/s. -> why + //InterlockedExchange((volatile LONG *)&Queue->Notify, (LONG)notificationEnabled); +} + +_Use_decl_annotations_ +VOID +EnlIndicateQueueState( + ENLP_QUEUE * Queue, + ENL_QUEUE_STATE State + ) +{ + Queue->State = State; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlCreateLink( + _In_ ULONG ProcessorIndex, + _In_ ULONG IdealNode, + _In_ BOOLEAN Poll, + _In_ BOOLEAN InitializePacketLayout, + _Out_ ENLP_LINK ** EnlLink + ) +{ + LogInformation(FLAG_DRIVER, L"ProcessorIndex=%u", ProcessorIndex); + + auto enlLink = wil::make_unique_nothrow<ENLP_LINK>(); + RETURN_NTSTATUS_IF( + STATUS_INSUFFICIENT_RESOURCES, + ! enlLink); + + enlLink->Ts = ReadTimeStampCounter(); + enlLink->Poll = Poll; + enlLink->InitializePacketLayout = InitializePacketLayout; + + RETURN_IF_NOT_STATUS_SUCCESS( + EnlpStartThread( + ProcessorIndex, + IdealNode, + EnlpIterationRoutine, + enlLink.get(), + enlLink->Poll ? NULL : &enlLink->ArmWaitEvent, + &enlLink->EnlThread)); + + *EnlLink = enlLink.release(); + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlIsPortActive( + _In_ ENLP_LINK * EnlLink, + _In_ ULONG PortIndex + ) +{ + ENLP_PORT* port = &EnlLink->Ports[PortIndex]; + ENLP_QUEUE* txq = &port->TxQueue[0]; + NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT); + + return (txq->Queue == nullptr) ? FALSE : TRUE; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlIsLinkActive( + _In_ ENLP_LINK * EnlLink + ) +{ + return (EnlIsPortActive(EnlLink, 0) || EnlIsPortActive(EnlLink, 1)); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlActivateLinkPort( + _In_ ENLP_LINK * EnlLink, + _In_ ULONG PortIndex, + _In_ ENL_INTERRUPT_ROUTINE Interrupt, + _In_ PVOID PortContext + ) +{ + ENLP_PORT* port = &EnlLink->Ports[PortIndex]; + + LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue); + + NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, PortIndex)); + NT_FRE_ASSERT(port->TxQueueCount == 0); + NT_FRE_ASSERT(port->RxQueueCount == 0); + + port->Interrupt = Interrupt; + port->PortContext = PortContext; + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlDeactivateLinkPort( + _In_ ENLP_LINK * EnlLink, + _In_ ULONG PortIndex + ) +{ + NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT); + + ENLP_PORT* port = &EnlLink->Ports[PortIndex]; + LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue); + + NT_FRE_ASSERT(EnlIsPortActive(EnlLink, PortIndex)); + + KLockThisExclusive(EnlLink->Lock); + NT_FRE_ASSERT(!EnlpIsThreadPaused(&EnlLink->EnlThread)); + EnlpPauseThread(&EnlLink->EnlThread); + +#if _KERNEL_MODE + KeFlushQueuedDpcs(); +#endif + + port->PortContext = NULL; + + //Clears reference to only first queue - Change for all queues + port->TxQueue[0].Queue = nullptr; + port->RxQueue[0].Queue = nullptr; + + + // As long as there's one port with active queues, the EnlThread will run. + for (ULONG i = 0; i < ENLP_PORT_COUNT; i++) + { + if (EnlIsPortActive(EnlLink, i)) + { + EnlpResumeThread(&EnlLink->EnlThread); + break; + } + } +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlCloseLink( + _In_ ENLP_LINK * EnlLink + ) +{ + ULONG i; + + for (i = 0; i < ENLP_PORT_COUNT; i++) + { + NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, i)); + } + + EnlpStopThread(&EnlLink->EnlThread); + + delete EnlLink; +} + +/////////////////////////////////////////////////////////////////////////////// +// Multi link wrapper APIs // +/////////////////////////////////////////////////////////////////////////////// + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlMCreateLink( + _In_range_(1, ENL_MLINK_MAX)ULONG LinkCount, + _In_reads_(LinkCount) ULONG ProcessorIndex, + _In_ BOOLEAN Poll, + _Out_ ENL_MLINK* EnlMLink + ) +{ + NTSTATUS status = STATUS_SUCCESS; + ULONG i = 0; + + RtlZeroMemory(EnlMLink, sizeof(*EnlMLink)); + + if (LinkCount < 1 || LinkCount > ENL_MLINK_MAX || + (LinkCount & (LinkCount - 1)) != 0) + { + status = STATUS_REQUEST_NOT_ACCEPTED; + goto exit; + } + + for (i = 0; i < LinkCount; i++) + { +#ifdef _KERNEL_MODE + const BOOLEAN initializePacketLayout = TRUE; +#else + const BOOLEAN initializePacketLayout = FALSE; +#endif + status = EnlCreateLink(ProcessorIndex, 0, Poll, initializePacketLayout, &EnlMLink->LinkHandle[i]); + + if (!NT_SUCCESS(status)) + { + goto exit; + } + } + + EnlMLink->LinkCount = LinkCount; + +exit: + + if (!NT_SUCCESS(status)) + { + for (; i > 0; i--) + { + EnlCloseLink(EnlMLink->LinkHandle[i - 1]); + EnlMLink->LinkHandle[i - 1] = NULL; + } + } + RETURN_NTSTATUS_IF( + status, + status != STATUS_SUCCESS); + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void EnlSetPdoWakeSignalCallback( + _In_ ENLP_LINK * EnlLink, + _In_ EVT_ENLP_PDO_WAKE_SIGNAL* evtPdoWakeSignal, + _In_ void* Context) +{ + EnlLink->EvtWakeSignal = evtPdoWakeSignal; + EnlLink->WakeSignalContext = Context; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlArmWake( + _In_ ENLP_LINK * EnlLink +) +{ + EnlLink->ArmedForWake = TRUE; + RtlZeroMemory(&EnlLink->WakeFrame[0], sizeof(EnlLink->WakeFrame)); + EnlLink->WakeFrameSize = 0; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlDisarmWake( + _In_ ENLP_LINK * EnlLink +) +{ + EnlLink->ArmedForWake = FALSE; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +size_t +EnlCopyWakeFrame( + _In_ ENLP_LINK * EnlLink, + _Out_writes_bytes_(BufferSize) unsigned char * Buffer, + _In_ size_t BufferSize +) +{ + if (EnlLink->WakeFrameSize == 0) + { + // There was no wake + return 0; + } + + if (BufferSize < EnlLink->WakeFrameSize) + { + // Wake frame is larger than what we can indicate + return 0; + } + + RtlCopyMemory(Buffer, &EnlLink->WakeFrame[0], EnlLink->WakeFrameSize); + auto const wakeFrameSize = EnlLink->WakeFrameSize; + + // Make sure to erase the wake frame, since the network interface might have + // multiple receive queues + RtlZeroMemory(&EnlLink->WakeFrame[0], EnlLink->WakeFrameSize); + EnlLink->WakeFrameSize = 0; + + return wakeFrameSize; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlMIsPortActive( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex + ) +{ + ULONG i; + + BOOLEAN result = EnlIsPortActive(EnlMLink->LinkHandle[0], PortIndex); + + for (i = 1; i < EnlMLink->LinkCount; i++) + { + NT_FRE_ASSERT(EnlIsPortActive(EnlMLink->LinkHandle[i], PortIndex) == result); + } + + return result; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlMIsLinkActive( + _In_ CONST ENL_MLINK* EnlMLink + ) +{ + ULONG i; + + BOOLEAN result = EnlIsLinkActive(EnlMLink->LinkHandle[0]); + + for (i = 1; i < EnlMLink->LinkCount; i++) + { + NT_FRE_ASSERT(EnlIsLinkActive(EnlMLink->LinkHandle[i]) == result); + } + + return result; +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlMActivateLinkPort( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex, + _In_ ENL_INTERRUPT_ROUTINE Interrupt, + _In_ PVOID PortContext + ) +{ + LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex); + + NTSTATUS status = STATUS_SUCCESS; + ULONG i; + + for (i = 0; i < EnlMLink->LinkCount; i++) + { + status = EnlActivateLinkPort( + EnlMLink->LinkHandle[i], + PortIndex, + Interrupt, + PortContext); + + if (!NT_SUCCESS(status)) + { + break; + } + } + + if (!NT_SUCCESS(status)) + { + for (; i > 0; i--) + { + EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex); + } + } + + RETURN_NTSTATUS_IF(status, + status != STATUS_SUCCESS); + + RETURN_STATUS_SUCCESS(); +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlMDeactivateLinkPort( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex + ) +{ + LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex); + + ULONG i; + + for (i = 0; i < EnlMLink->LinkCount; i++) + { + EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex); + } +} + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlMCloseLink( + _Inout_ ENL_MLINK* EnlMLink + ) +{ + LogInformation(FLAG_DRIVER, L"EnlMLink=%p", EnlMLink); + + ULONG i; + + for (i = 0; i < EnlMLink->LinkCount; i++) + { + EnlCloseLink(EnlMLink->LinkHandle[i]); + EnlMLink->LinkHandle[i] = NULL; + } + + EnlMLink->LinkCount = 0; +} diff --git a/network/netadaptercx/netvadapterlibrary/code/enl.h b/network/netadaptercx/netvadapterlibrary/code/enl.h new file mode 100644 index 00000000..2927a582 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/enl.h @@ -0,0 +1,264 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +// +// Emulated Network Link (ENL) definitions +// + +// +// An ENL connects two virtual network adapters directly. Packets sent over +// one adapter are delivered to the other adapter and vice versa. +// +// An ENL is created with a processor index. This processor is used to emulate +// the NIC hardware for the adapters connected by the ENL. +// +// Caller can send packets as NBLs over a given port, and also receive +// incoming packets placed into NBLs over a given port. +// +// ENL indicates tx and rx NBL completions over the target processor(s) +// determined by the RSS indirection table and the hash value for each NBL. +// The current version of the ENL requires a symmetric Toeplitz hash key +// to be used by the system globally so that both directions of a given +// 4-tuple (or 2-tuple) yield the same hash value. ENL currently indicates +// NBL completions by queueing DPCs to the target processor. +// + + +#ifndef _KERNEL_MODE +#define ASSERT(x) NT_ASSERT(x) + +#endif +#include "rtl/KWaitEvent.h" +#include "rtl/KPushLock.h" + +#define ENL_MAX_PROC_COUNT 16 +#define ENLP_PORT_COUNT 2 + +#define TX TRUE +#define RX FALSE + +enum ENL_QUEUE_STATE { + Stopped, + Started + }; + +typedef +VOID +(ENL_INTERRUPT_ROUTINE) ( + _Inout_ PVOID PortContext, + _In_ bool TxRx + ); + +struct ENLP_LINK; +struct DECLSPEC_ALIGN(PAGE_SIZE) ENLP_PORT; +struct ENLP_QUEUE; + +typedef NTSTATUS(EVT_ENLP_PDO_WAKE_SIGNAL)(_In_ void* Context); + +_IRQL_requires_max_(PASSIVE_LEVEL) +ENLP_QUEUE * +EnlCreateQueue( + _In_ NETPACKETQUEUE Queue, + _In_ BOOLEAN Tx + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlDestroyQueue( + _In_ ENLP_QUEUE * QueueContext, + _In_ BOOLEAN Tx + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlRingDoorBell( + _In_ ENLP_QUEUE * QueueContext, + _In_ ULONG EndIndex + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlArmInterrupt( + _In_ ENLP_QUEUE * QueueContext, + _In_ BOOLEAN NotificationEnabled + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlSetPdoWakeSignalCallback( + _In_ ENLP_LINK * EnlLinkHandle, + _In_ EVT_ENLP_PDO_WAKE_SIGNAL* evtPdoWakeSignal, + _In_ void* Context + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlArmWake( + _In_ ENLP_LINK * EnlLinkHandle + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlDisarmWake( + _In_ ENLP_LINK * EnlLinkHandle + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +size_t +EnlCopyWakeFrame( + _In_ ENLP_LINK * EnlLinkHandle, + _Out_writes_bytes_(BufferSize) unsigned char * Buffer, + _In_ size_t BufferSize + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlIsLinkActive( + _In_ ENLP_LINK * EnlLinkHandle + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlIndicateQueueState( + _In_ ENLP_QUEUE * EnlLinkHandle, + _In_ ENL_QUEUE_STATE State + ); + +/////////////////////////////////////////////////////////////////////////////// +// Multi link wrapper APIs // +/////////////////////////////////////////////////////////////////////////////// + +#define ENL_MLINK_MAX 4 + +typedef struct +{ + _Field_range_(1, ENL_MLINK_MAX) ULONG LinkCount; + _Field_size_(LinkCount) ENLP_LINK * LinkHandle[ENL_MLINK_MAX]; +} ENL_MLINK; + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlMCreateLink( + _In_range_(1, ENL_MLINK_MAX)ULONG LinkCount, + _In_reads_(LinkCount) ULONG ProcessorIndex, + _In_ BOOLEAN Poll, + _Out_ ENL_MLINK* EnlMLink + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +BOOLEAN +EnlMIsPortActive( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +NTSTATUS +EnlMActivateLinkPort( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex, + _In_ ENL_INTERRUPT_ROUTINE Interrupt, + _In_ PVOID PortContext + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +void +EnlMDeactivateLinkPort( + _In_ CONST ENL_MLINK* EnlMLink, + _In_ ULONG PortIndex + ); + +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +EnlMCloseLink( + _Inout_ ENL_MLINK* EnlMLink + ); + +#define ENL_MAXIMUM_WAKE_FRAME_SIZE 1514 + +typedef +_IRQL_requires_max_(PASSIVE_LEVEL) +VOID +(ENLP_ITERATION_ROUTINE) ( + _In_ PVOID IterationContext + ); + +struct ENLP_THREAD_STATE +{ + BOOLEAN PauseRequested{}; + BOOLEAN StopRequested{}; + ENLP_ITERATION_ROUTINE* IterationRoutine{}; + PVOID IterationContext{}; + KAutoEvent *ArmWaitEvent{}; + KAutoEvent ResumeEvent{}; + KAutoEvent PausingEvent{}; + ULONG ProcIndex{}; + ULONG IdealNode{}; + unique_thread Thread{}; +}; + +class NetvQueue; +class NetvRxQueue; +class NetvTxQueue; + +struct ENLP_QUEUE +{ + BOOLEAN TxRx{}; + BOOLEAN Notify{}; + ULONG QueueEnd{}; + ULONG QueueNext{}; + + ENLP_PORT * EnlPortHandle{}; + + BOOLEAN Armed{}; + WDFSPINLOCK Spinlock{}; + KAutoEvent *ArmWaitEvent{}; + + ENL_QUEUE_STATE State{}; + + NetvQueue * Queue{nullptr}; + union { + NetvRxQueue * RxQueue; + NetvTxQueue * TxQueue; + }; + +}; + +struct DECLSPEC_ALIGN(PAGE_SIZE) ENLP_PORT +{ + ENL_INTERRUPT_ROUTINE* Interrupt; + PVOID PortContext; + ULONG TxQueueCount; + ULONG RxQueueCount; + + DECLSPEC_CACHEALIGN + ENLP_QUEUE TxQueue[ENL_MAX_PROC_COUNT]; + + DECLSPEC_CACHEALIGN + ENLP_QUEUE RxQueue[ENL_MAX_PROC_COUNT]; +}; + +struct ENLP_LINK +{ + KPushLock Lock{}; + ULONG64 Ts{}; + ULONG64 BusyTicks{}; + ULONG64 EmptyTicks{}; + BOOLEAN Poll{}; + BOOLEAN InitializePacketLayout{}; + + // + // Power related features. If PowerInterface is NULL none of the other + // fields have meaning + // + EVT_ENLP_PDO_WAKE_SIGNAL* EvtWakeSignal; + void* WakeSignalContext; + BOOLEAN ArmedForWake{}; + unsigned char WakeFrame[ENL_MAXIMUM_WAKE_FRAME_SIZE]; + size_t WakeFrameSize{}; + + KAutoEvent ArmWaitEvent{}; + ENLP_THREAD_STATE EnlThread{}; + ENLP_PORT Ports[ENLP_PORT_COUNT]; +}; +#define MAX_ADAPTER_COUNT 2 +extern ENL_MLINK NetvEnlMLink[MAX_ADAPTER_COUNT / 2]; +C_ASSERT((ENLP_PORT_COUNT & 0x1) == 0); diff --git a/network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp b/network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp new file mode 100644 index 00000000..ffe70d6e --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp @@ -0,0 +1,136 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#include "pch.hpp" +//#include <ntassert.h> +#include "enlthreads.h" + +#ifdef _KERNEL_MODE +unique_thread::operator bool( + void +) const +{ + return !!NtHandle; +} + +void unique_thread::reset( +) +{ + NtHandle.reset(); + ObHandle.reset(); +} +#endif + +ENL_THREAD +EnlGetCurrentThread( + void +) +{ +#ifdef _KERNEL_MODE + return KeGetCurrentThread(); +#else + return GetCurrentThreadId(); +#endif +} + +_Use_decl_annotations_ +NTSTATUS +EnlThreadCreate( + ENL_START_ROUTINE StartRoutine, + void * Context, + unique_thread & Thread +) +{ +#ifdef _KERNEL_MODE + + unique_zw_handle ntHandle; + unique_pkthread obHandle; + + auto const ntStatus = PsCreateSystemThread( + &ntHandle, + THREAD_ALL_ACCESS, + nullptr, + nullptr, + nullptr, + StartRoutine, + Context); + + if (ntStatus != STATUS_SUCCESS) + { + return ntStatus; + } + + NT_FRE_ASSERT( + NT_SUCCESS( + ObReferenceObjectByHandle( + ntHandle.get(), + THREAD_ALL_ACCESS, + nullptr, + KernelMode, + reinterpret_cast<void **>(&obHandle), + nullptr))); + + Thread.NtHandle = wistd::move(ntHandle); + Thread.ObHandle = wistd::move(obHandle); + +#else + wil::unique_handle thread{ CreateThread(nullptr, 0, StartRoutine, Context, 0, nullptr) }; + + if (!thread) + { + return NTSTATUS_FROM_WIN32(GetLastError()); + } + + Thread = wistd::move(thread); +#endif + + return STATUS_SUCCESS; +} + +void +EnlThreadSetPriority( + unique_thread & Thread, + ENL_THREAD_PRIORITY Priority +) +{ +#ifdef _KERNEL_MODE + // KeSetBasePriorityThread does not take the actual priority, but an increment + // to be added to the current base priority. Calculate this value. + auto const increment = Priority - (LOW_REALTIME_PRIORITY + LOW_PRIORITY) / 2; + KeSetBasePriorityThread(Thread.ObHandle.get(), increment); +#else + SetThreadPriority(Thread.get(), Priority); +#endif +} + +_Use_decl_annotations_ +void +EnlThreadSetAffinity( + PGROUP_AFFINITY GroupAffinity, + PGROUP_AFFINITY PreviousAffinity +) +{ +#ifdef _KERNEL_MODE + KeSetSystemGroupAffinityThread(GroupAffinity, PreviousAffinity); +#else + SetThreadGroupAffinity(GetCurrentThread(), GroupAffinity, PreviousAffinity); +#endif +} + +_Use_decl_annotations_ +void +EnlThreadWaitForTermination( + unique_thread & Thread +) +{ +#ifdef _KERNEL_MODE + KeWaitForSingleObject( + Thread.ObHandle.get(), + KWAIT_REASON::Executive, + KernelMode, + FALSE, + nullptr); +#else + WaitForSingleObject( + Thread.get(), + INFINITE); +#endif +} diff --git a/network/netadaptercx/netvadapterlibrary/code/enlthreads.h b/network/netadaptercx/netvadapterlibrary/code/enlthreads.h new file mode 100644 index 00000000..062d728e --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/enlthreads.h @@ -0,0 +1,119 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#pragma once + +#include <wil/resource.h> +// #include <KMacros.h> + +#ifdef _KERNEL_MODE + using ENL_START_ROUTINE = KSTART_ROUTINE; + using ENL_THREAD_ROUTINE_RETURN = VOID; + using ENL_THREAD = PKTHREAD; + // static const EC_THREAD EC_THREAD_INVALID = nullptr; + using ENL_THREAD_PRIORITY = LONG; + // #define EC_INVALID_THREAD_PRIORITY MAXLONG + + using unique_zw_handle = wil::unique_any<HANDLE, decltype(&::ZwClose), &::ZwClose>; + using unique_pkthread = wil::unique_any<PKTHREAD, decltype(&ObfDereferenceObject), &ObfDereferenceObject>; + using unique_pkevent = wil::unique_any<PKEVENT, decltype(&ObfDereferenceObject), &ObfDereferenceObject>; + using unique_completionport = wil::unique_any<void *, decltype(&ObfDereferenceObject), &ObfDereferenceObject>; + + struct unique_thread + { + unique_zw_handle + NtHandle; + + unique_pkthread + ObHandle; + + operator bool( + void + ) const; + + void reset( + ); + }; +#else + +#ifndef NOTHING +#define NOTHING +#endif + +#ifndef AFFINITY_MASK +#define AFFINITY_MASK(n) ((KAFFINITY)1 << (n)) +#endif + +typedef struct _NDTBUS_POWER_INTERFACE_STANDARD +{ +} NDTBUS_POWER_INTERFACE_STANDARD; + +_IRQL_requires_same_ +typedef DWORD (WINAPI ENL_START_ROUTINE)( + LPVOID lpThreadParameter +); +using ENL_THREAD_ROUTINE_RETURN = DWORD; +using ENL_THREAD = DWORD; +// static const EC_THREAD EC_THREAD_INVALID = 0; +using ENL_THREAD_PRIORITY = int; +// #define EC_INVALID_THREAD_PRIORITY THREAD_PRIORITY_ERROR_RETURN +using unique_thread = wil::unique_handle; + +#define KeGetProcessorIndexFromNumber(_processor) (_processor)->Number +#define KeMemoryBarrier() MemoryBarrier() + +inline +NTSTATUS +KeGetProcessorNumberFromIndex ( + _In_ ULONG ProcIndex, + _Out_ PPROCESSOR_NUMBER ProcNumber + ) +{ + ProcNumber->Number = static_cast<UCHAR>(ProcIndex); + ProcNumber->Group = 0; + ProcNumber->Reserved = 0; + + return STATUS_SUCCESS; +} + +inline +VOID +KeQueryNodeActiveAffinity ( + __in USHORT NodeNumber, + __out_opt PGROUP_AFFINITY Affinity, + __out_opt PUSHORT Count +) +{ + UNREFERENCED_PARAMETER(NodeNumber); + UNREFERENCED_PARAMETER(Count); + + GetThreadGroupAffinity(GetCurrentThread(), Affinity); +} +#endif + +ENL_THREAD +EnlGetCurrentThread( + void +); + +NTSTATUS +EnlThreadCreate( + _In_ ENL_START_ROUTINE StartRoutine, + _In_opt_ void * Context, + _Out_ unique_thread & Thread +); + +void +EnlThreadSetPriority( + _In_ unique_thread & Thread, + _In_ ENL_THREAD_PRIORITY Priority +); + +void +EnlThreadSetAffinity( + _In_ PGROUP_AFFINITY GroupAffinity, + _Out_opt_ PGROUP_AFFINITY PreviousAffinity +); + +void +EnlThreadWaitForTermination( + _In_ unique_thread & Thread +); diff --git a/network/netadaptercx/netvadapterlibrary/code/memory.cpp b/network/netadaptercx/netvadapterlibrary/code/memory.cpp new file mode 100644 index 00000000..35a3f2c5 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/memory.cpp @@ -0,0 +1,11 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#include "pch.hpp" +#include "memory.h" +#include "memory.tmh" + +// for wil::make_unique_nothrow +void* +operator new(size_t s, std::nothrow_t const&) +{ + return operator new(s); +}
\ No newline at end of file diff --git a/network/netadaptercx/netvadapterlibrary/code/memory.h b/network/netadaptercx/netvadapterlibrary/code/memory.h new file mode 100644 index 00000000..1200c72c --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/memory.h @@ -0,0 +1,4 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#pragma once + +#define MAX_RX_BUFFER_SIZE 65535 diff --git a/network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h b/network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h new file mode 100644 index 00000000..b5e34518 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h @@ -0,0 +1,85 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. + +#pragma once + +#include <net/virtualaddress.h> + +// +// Following are some helper APIs for common ring manipulations. +// They are all implemented using iterator +// + +inline +SIZE_T +GetTxPacketDataLength( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + SIZE_T length = 0; + + for (NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(Iterator); + NetFragmentIteratorHasAny(&fi); + NetFragmentIteratorAdvance(&fi)) + { + NET_FRAGMENT* fragment = NetFragmentIteratorGetFragment(&fi); + length += (SIZE_T)fragment->ValidLength; + } + + return length; +} + +inline +VOID +CompleteTxPacketsBatch( + _In_ NET_RING_COLLECTION const* Rings, + _In_ UINT32 BatchSize +) +{ + UINT32 packetCount = 0; + + NET_RING_PACKET_ITERATOR pi = NetRingGetDrainPackets(Rings); + + while (NetPacketIteratorHasAny(&pi)) + { + NET_PACKET* packet = NetPacketIteratorGetPacket(&pi); + + // this function uses Scratch field as the bit for testing completion + if (!packet->Scratch) + { + break; + } + + packetCount++; + + NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(&pi); + NetFragmentIteratorAdvanceToTheEnd(&fi); + + NetPacketIteratorAdvance(&pi); + + if (packetCount >= BatchSize) + { + NetPacketIteratorSet(&pi); + Rings->Rings[NetRingTypeFragment]->BeginIndex = NetFragmentIteratorGetIndex(&fi); + } + } +} + +inline +void +CancelRxPackets( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING_PACKET_ITERATOR pi = NetRingGetAllPackets(Rings); + + for (; NetPacketIteratorHasAny(&pi); NetPacketIteratorAdvance(&pi)) + { + NetPacketIteratorGetPacket(&pi)->Ignore = 1; + } + + NetPacketIteratorSet(&pi); + + NET_RING_FRAGMENT_ITERATOR fi = NetRingGetAllFragments(Rings); + NetFragmentIteratorAdvanceToTheEnd(&fi); + NetFragmentIteratorSet(&fi); +} diff --git a/network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h b/network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h new file mode 100644 index 00000000..fb010751 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h @@ -0,0 +1,281 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. + +#pragma once + +#include <net/ringcollection.h> + +typedef struct _NET_RING_ITERATOR +{ + + NET_RING_COLLECTION const* + Rings; + + UINT32* const + IndexToSet; + + UINT32 + Index; + + UINT32 const + End; + +} NET_RING_ITERATOR; + +typedef struct _NET_RING_PACKET_ITERATOR +{ + + NET_RING_ITERATOR + Iterator; + +} NET_RING_PACKET_ITERATOR; + +typedef struct _NET_RING_FRAGMENT_ITERATOR +{ + + NET_RING_ITERATOR + Iterator; + +} NET_RING_FRAGMENT_ITERATOR; + + +inline +NET_RING_PACKET_ITERATOR +NetRingGetPostPackets( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING* ring = Rings->Rings[NetRingTypePacket]; + NET_RING_PACKET_ITERATOR iterator = { + Rings, &ring->NextIndex, ring->NextIndex, ring->EndIndex, + }; + + return iterator; +} + +inline +NET_RING_PACKET_ITERATOR +NetRingGetDrainPackets( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING* ring = Rings->Rings[NetRingTypePacket]; + NET_RING_PACKET_ITERATOR iterator = { + Rings, &ring->BeginIndex, ring->BeginIndex, ring->NextIndex, + }; + + return iterator; +} + +inline +NET_RING_PACKET_ITERATOR +NetRingGetAllPackets( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING* ring = Rings->Rings[NetRingTypePacket]; + NET_RING_PACKET_ITERATOR iterator = { + Rings, &ring->BeginIndex, ring->BeginIndex, ring->EndIndex, + }; + + return iterator; +} + +inline +NET_PACKET* +NetPacketIteratorGetPacket( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + return NetRingGetPacketAtIndex( + Iterator->Iterator.Rings->Rings[NetRingTypePacket], + Iterator->Iterator.Index); +} + +inline +UINT32 +NetPacketIteratorGetIndex( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + return Iterator->Iterator.Index; +} + +inline +BOOLEAN +NetPacketIteratorHasAny( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + return Iterator->Iterator.Index != Iterator->Iterator.End; +} + +inline +UINT32 +NetPacketIteratorGetCount( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + NET_RING const* ring = Iterator->Iterator.Rings->Rings[NetRingTypePacket]; + + return (Iterator->Iterator.End - Iterator->Iterator.Index) & ring->ElementIndexMask; +} + +inline +void +NetPacketIteratorAdvance( + _In_ NET_RING_PACKET_ITERATOR* Iterator +) +{ + Iterator->Iterator.Index = NetRingIncrementIndex( + Iterator->Iterator.Rings->Rings[NetRingTypePacket], + Iterator->Iterator.Index); +} + +inline +void +NetPacketIteratorAdvanceToTheEnd( + _In_ NET_RING_PACKET_ITERATOR* Iterator +) +{ + Iterator->Iterator.Index = Iterator->Iterator.End; +} + +inline +void +NetPacketIteratorSet( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + *Iterator->Iterator.IndexToSet + = Iterator->Iterator.Index; +} + + +inline +NET_RING_FRAGMENT_ITERATOR +NetPacketIteratorGetFragments( + _In_ NET_RING_PACKET_ITERATOR const* Iterator +) +{ + NET_RING const* ring = Iterator->Iterator.Rings->Rings[NetRingTypeFragment]; + NET_PACKET const* packet = NetPacketIteratorGetPacket(Iterator); + UINT32 const end = NetRingIncrementIndex(ring, + packet->FragmentIndex + packet->FragmentCount - 1); + NET_RING_FRAGMENT_ITERATOR iterator = { + Iterator->Iterator.Rings, NULL, packet->FragmentIndex, end, + }; + + return iterator; +} + +inline +NET_RING_FRAGMENT_ITERATOR +NetRingGetPostFragments( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING* ring = Rings->Rings[NetRingTypeFragment]; + NET_RING_FRAGMENT_ITERATOR iterator = { + Rings, &ring->NextIndex, ring->NextIndex, ring->EndIndex, + }; + + return iterator; +} + +inline +NET_RING_FRAGMENT_ITERATOR +NetRingGetDrainFragments( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING* ring = Rings->Rings[NetRingTypeFragment]; + NET_RING_FRAGMENT_ITERATOR iterator = { + Rings, &ring->BeginIndex, ring->BeginIndex, ring->NextIndex, + }; + + return iterator; +} + +inline +NET_RING_FRAGMENT_ITERATOR +NetRingGetAllFragments( + _In_ NET_RING_COLLECTION const* Rings +) +{ + NET_RING* ring = Rings->Rings[NetRingTypeFragment]; + NET_RING_FRAGMENT_ITERATOR iterator = { + Rings, &ring->BeginIndex, ring->BeginIndex, ring->EndIndex, + }; + + return iterator; +} + +inline +NET_FRAGMENT* +NetFragmentIteratorGetFragment( + _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator +) +{ + return NetRingGetFragmentAtIndex( + Iterator->Iterator.Rings->Rings[NetRingTypeFragment], + Iterator->Iterator.Index); +} + +inline +UINT32 +NetFragmentIteratorGetIndex( + _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator +) +{ + return Iterator->Iterator.Index; +} + +inline +BOOLEAN +NetFragmentIteratorHasAny( + _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator +) +{ + return Iterator->Iterator.Index != Iterator->Iterator.End; +} + +inline +UINT32 +NetFragmentIteratorGetCount( + _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator +) +{ + NET_RING const* ring = Iterator->Iterator.Rings->Rings[NetRingTypeFragment]; + + return (Iterator->Iterator.End - Iterator->Iterator.Index) & ring->ElementIndexMask; +} + +inline +void +NetFragmentIteratorAdvance( + _In_ NET_RING_FRAGMENT_ITERATOR* Iterator +) +{ + Iterator->Iterator.Index = NetRingIncrementIndex( + Iterator->Iterator.Rings->Rings[NetRingTypeFragment], + Iterator->Iterator.Index); +} + +inline +void +NetFragmentIteratorAdvanceToTheEnd( + _In_ NET_RING_FRAGMENT_ITERATOR* Iterator +) +{ + Iterator->Iterator.Index = Iterator->Iterator.End; +} + +inline +void +NetFragmentIteratorSet( + _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator +) +{ + *(Iterator->Iterator.IndexToSet) + = Iterator->Iterator.Index; +} diff --git a/network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h b/network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h new file mode 100644 index 00000000..a6a019e4 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h @@ -0,0 +1,268 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Module Name: + + xfilter.h + +Abstract: + + Header file for the address filtering library for NDIS MAC's. + +Author: + +Environment: + +Notes: + + None. + +Revision History: + +--*/ + +#ifndef _X_FILTER_DEFS_ +#define _X_FILTER_DEFS_ + +#pragma once + +#define ETH_LENGTH_OF_ADDRESS 6 + + +// +// ZZZ This is a little-endian specific check. +// +#define ETH_IS_MULTICAST(Address) \ + (BOOLEAN)(((PUCHAR)(Address))[0] & ((UCHAR)0x01)) + + +// +// Check whether an address is broadcast. +// +#define ETH_IS_BROADCAST(Address) \ + ((((PUCHAR)(Address))[0] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[1] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[2] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[3] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[4] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[5] == ((UCHAR)0xff))) + + +// +// This macro will compare network addresses. +// +// A - Is a network address. +// +// B - Is a network address. +// +// Result - The result of comparing two network address. +// +// Result < 0 Implies the B address is greater. +// Result > 0 Implies the A element is greater. +// Result = 0 Implies equality. +// +// Note that this is an arbitrary ordering. There is not +// defined relation on network addresses. This is ad-hoc! +// +// +#define ETH_COMPARE_NETWORK_ADDRESSES(_A, _B, _Result) \ +{ \ + if (*(ULONG UNALIGNED *)&(_A)[2] > \ + *(ULONG UNALIGNED *)&(_B)[2]) \ + { \ + *(_Result) = 1; \ + } \ + else if (*(ULONG UNALIGNED *)&(_A)[2] < \ + *(ULONG UNALIGNED *)&(_B)[2]) \ + { \ + *(_Result) = (UINT)-1; \ + } \ + else if (*(USHORT UNALIGNED *)(_A) > \ + *(USHORT UNALIGNED *)(_B)) \ + { \ + *(_Result) = 1; \ + } \ + else if (*(USHORT UNALIGNED *)(_A) < \ + *(USHORT UNALIGNED *)(_B)) \ + { \ + *(_Result) = (UINT)-1; \ + } \ + else \ + { \ + *(_Result) = 0; \ + } \ +} + +// +// This macro will compare network addresses. +// +// A - Is a network address. +// +// B - Is a network address. +// +// Result - The result of comparing two network address. +// +// Result != 0 Implies inequality. +// Result == 0 Implies equality. +// +// +#define ETH_COMPARE_NETWORK_ADDRESSES_EQ(_A,_B, _Result) \ +{ \ + if ((*(ULONG UNALIGNED *)&(_A)[2] == \ + *(ULONG UNALIGNED *)&(_B)[2]) && \ + (*(USHORT UNALIGNED *)(_A) == \ + *(USHORT UNALIGNED *)(_B))) \ + { \ + *(_Result) = 0; \ + } \ + else \ + { \ + *(_Result) = 1; \ + } \ +} + + +// +// This macro is used to copy from one network address to +// another. +// +#define ETH_COPY_NETWORK_ADDRESS(_D, _S) \ +{ \ + *((ULONG UNALIGNED *)(_D)) = *((ULONG UNALIGNED *)(_S)); \ + *((USHORT UNALIGNED *)((UCHAR *)(_D)+4)) = *((USHORT UNALIGNED *)((UCHAR *)(_S)+4)); \ +} + +#define TR_LENGTH_OF_FUNCTIONAL 4 +#define TR_LENGTH_OF_ADDRESS 6 + + +// +// Only the low 32 bits of the functional/group address +// are needed since the upper 16 bits is always c0-00. +// +typedef ULONG TR_FUNCTIONAL_ADDRESS; +typedef ULONG TR_GROUP_ADDRESS; + + +#define TR_IS_NOT_DIRECTED(_Address, _Result) \ +{ \ + *(_Result) = (BOOLEAN)((_Address)[0] & 0x80); \ +} + +#define TR_IS_FUNCTIONAL(_Address, _Result) \ +{ \ + *(_Result) = (BOOLEAN)(((_Address)[0] & 0x80) && \ + !((_Address)[2] & 0x80)); \ +} + +// +// +#define TR_IS_GROUP(_Address, _Result) \ +{ \ + *(_Result) = (BOOLEAN)((_Address)[0] & (_Address)[2] & 0x80); \ +} + +// +// +#define TR_IS_SOURCE_ROUTING(_Address, _Result) \ +{ \ + *(_Result) = (BOOLEAN)((_Address)[0] & 0x80); \ +} + +// +// Check for NDIS_PACKET_TYPE_MAC_FRAME +// +#define TR_IS_MAC_FRAME(_PacketHeader) ((((PUCHAR)_PacketHeader)[1] & 0xFC) == 0) + + +// +// Check whether an address is broadcast. This is a little-endian check. +// +#define TR_IS_BROADCAST(_Address, _Result) \ +{ \ + *(_Result) = (BOOLEAN)(((*(UNALIGNED USHORT *)&(_Address)[0] == 0xFFFF) || \ + (*(UNALIGNED USHORT *)&(_Address)[0] == 0x00C0)) && \ + (*(UNALIGNED ULONG *)&(_Address)[2] == 0xFFFFFFFF));\ +} + + +// +// This macro will compare network addresses. +// +// A - Is a network address. +// +// B - Is a network address. +// +// Result - The result of comparing two network address. +// +// Result < 0 Implies the B address is greater. +// Result > 0 Implies the A element is greater. +// Result = 0 Implies equality. +// +// Note that this is an arbitrary ordering. There is not +// defined relation on network addresses. This is ad-hoc! +// +// +#define TR_COMPARE_NETWORK_ADDRESSES(_A, _B, _Result) \ +{ \ + if (*(ULONG UNALIGNED *)&(_A)[2] > \ + *(ULONG UNALIGNED *)&(_B)[2]) \ + { \ + *(_Result) = 1; \ + } \ + else if (*(ULONG UNALIGNED *)&(_A)[2] < \ + *(ULONG UNALIGNED *)&(_B)[2]) \ + { \ + *(_Result) = (UINT)-1; \ + } \ + else if (*(USHORT UNALIGNED *)(_A) > \ + *(USHORT UNALIGNED *)(_B)) \ + { \ + *(_Result) = 1; \ + } \ + else if (*(USHORT UNALIGNED *)(_A) < \ + *(USHORT UNALIGNED *)(_B)) \ + { \ + *(_Result) = (UINT)-1; \ + } \ + else \ + { \ + *(_Result) = 0; \ + } \ +} + +// +// This macro will compare network addresses. +// +// A - Is a network address. +// +// B - Is a network address. +// +// Result - The result of comparing two network address. +// +// Result != 0 Implies inequality. +// Result == 0 Implies equality. +// +// +#define TR_COMPARE_NETWORK_ADDRESSES_EQ(_A, _B, _Result) \ +{ \ + if ((*(ULONG UNALIGNED *)&(_A)[2] == *(ULONG UNALIGNED *)&(_B)[2]) && \ + (*(USHORT UNALIGNED *)&(_A)[0] == *(USHORT UNALIGNED *)&(_B)[0])) \ + { \ + *(_Result) = 0; \ + } \ + else \ + { \ + *(_Result) = 1; \ + } \ +} + + +// +// This macro is used to copy from one network address to +// another. +// +#define TR_COPY_NETWORK_ADDRESS(_D, _S) \ +{ \ + *((ULONG UNALIGNED *)(_D)) = *((ULONG UNALIGNED *)(_S)); \ + *((USHORT UNALIGNED *)((UCHAR *)(_D)+4)) = \ + *((USHORT UNALIGNED *)((UCHAR *)(_S)+4)); \ +} + +#endif // _X_FILTER_DEFS_ diff --git a/network/netadaptercx/netvadapterlibrary/code/pch.hpp b/network/netadaptercx/netvadapterlibrary/code/pch.hpp new file mode 100644 index 00000000..cf2f0f22 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/pch.hpp @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved +#pragma once + +#include <initguid.h> + +#ifndef _KERNEL_MODE +// This is a user-mode driver +#include <windows.h> + +#else +// This is a kernel-mode driver +#include <ntddk.h> +#define NTSTRSAFE_LIB +#include <ntstrsafe.h> +#endif + +// This is a common WDF header (for both KMDF and UMDF) +#include <wdf.h> + +#include <netadaptercx.h> +//#include <wdftriage.h> +#include "net/netringiterator.h" +#include "net/netpacketlibrary.h" +#include <net/rsc.h> +#include <net/gso.h> +#include <net/checksum.h> +#include <net/databuffer.h> +#include <net/returncontext.h> + +#include "enlthreads.h" +#include "enl.h" + +#include "trace.h" + diff --git a/network/netadaptercx/netvadapterlibrary/code/queue.cpp b/network/netadaptercx/netvadapterlibrary/code/queue.cpp new file mode 100644 index 00000000..667d59f7 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/queue.cpp @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +#include "pch.hpp" + +#include <net/ringcollection.h> + +#include "queue.h" + +NetvQueue::NetvQueue( + NETPACKETQUEUE Handle, + NetvAdapter & Adapter, + NET_RING_COLLECTION const * Rings +) + : m_handle{Handle} + , m_adapter{Adapter} + , m_rings{Rings} +{ +} + +NET_RING * +NetvQueue::GetPacketRing( + void +) +{ + return NetRingCollectionGetPacketRing(m_rings); +} + +NET_RING * +NetvQueue::GetFragmentRing( + void +) +{ + return NetRingCollectionGetFragmentRing(m_rings); +} diff --git a/network/netadaptercx/netvadapterlibrary/code/queue.h b/network/netadaptercx/netvadapterlibrary/code/queue.h new file mode 100644 index 00000000..becb72ce --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/queue.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +#pragma once + +class NetvAdapter; +struct ENLP_QUEUE; + +class NetvQueue +{ + +public: + + NetvQueue( + NETPACKETQUEUE Handle, + NetvAdapter & Adapter, + NET_RING_COLLECTION const * Rings + ); + + NET_RING * + GetPacketRing( + void + ); + + NET_RING * + GetFragmentRing( + void + ); + + NETPACKETQUEUE const + m_handle{WDF_NO_HANDLE}; + + NetvAdapter & + m_adapter; + + NET_RING_COLLECTION const * + m_rings; + + ENLP_QUEUE * + EnlQueueHandle; + +}; diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h new file mode 100644 index 00000000..0bbedadb --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h @@ -0,0 +1,67 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#pragma once + +#include <KMacros.h> + +class KCriticalRegion +{ +public: + + PAGED KCriticalRegion() : m_Entered(false) { } + + PAGED ~KCriticalRegion() { if (m_Entered) Leave(); } + + KCriticalRegion(KCriticalRegion &) = delete; + KCriticalRegion &operator=(KCriticalRegion &) = delete; + + PAGED void Enter() + { + ASSERT(m_Entered == false); + UnbalancedEnter(); + m_Entered = true; + } + + PAGED void Leave() + { + ASSERT(m_Entered == true); + m_Entered = false; + UnbalancedLeave(); + } + + static PAGED void UnbalancedEnter() + { +#if _KERNEL_MODE + KeEnterCriticalRegion(); +#endif + } + + static PAGED void UnbalancedLeave() + { +#if _KERNEL_MODE + KeLeaveCriticalRegion(); +#endif + } + +private: + + bool m_Entered; +}; + +struct KDefaultRegion +{ + void Enter() { } + void Leave() { } +}; + +struct KIrqlRegion +{ + KIrqlRegion() { } + ~KIrqlRegion() { } + + void Enter() { } + void Leave() { } + + KIRQL m_OldIrql; +}; + + diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h new file mode 100644 index 00000000..d398f137 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h @@ -0,0 +1,114 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#pragma once + +#include <KCriticalRegion.h> +#include <KPushLock.h> + +class KRTL_CLASS KLockHolder +{ +private: + + enum { Unlocked, Shared, Exclusive } m_State; + +public: + + PAGED KLockHolder(KPushLockBase &lock) : m_Lock(lock), m_State(Unlocked) { } + PAGED ~KLockHolder() + { + switch (m_State) + { + case Shared: + ReleaseShared(); + break; + case Exclusive: + ReleaseExclusive(); + break; + } + } + + KLockHolder(KLockHolder &) = delete; + KLockHolder &operator=(KLockHolder &) = delete; + + _IRQL_requires_(PASSIVE_LEVEL) + PAGED void AcquireShared() + { + m_Region.Enter(); + ASSERT(m_State == Unlocked); + m_Lock.AcquireShared(); + m_State = Shared; + } + + _IRQL_requires_(PASSIVE_LEVEL) + PAGED void ReleaseShared() + { + ASSERT(m_State == Shared); + m_Lock.ReleaseShared(); + m_State = Unlocked; + m_Region.Leave(); + } + + _IRQL_requires_(PASSIVE_LEVEL) + PAGED void AcquireExclusive() + { + m_Region.Enter(); + ASSERT(m_State == Unlocked); + m_Lock.AcquireExclusive(); + m_State = Exclusive; + } + + _IRQL_requires_(PASSIVE_LEVEL) + PAGED void ReleaseExclusive() + { + ASSERT(m_State == Exclusive); + m_Lock.ReleaseExclusive(); + m_State = Unlocked; + m_Region.Leave(); + } + +private: + + KPushLockBase &m_Lock; + KCriticalRegion m_Region; +}; + +class KRTL_CLASS KLockThisShared : protected KLockHolder +{ +public: + + PAGED KLockThisShared(KPushLockBase &lock) : KLockHolder(lock) + { + AcquireShared(); + } + + PAGED void Acquire() + { + AcquireShared(); + } + + PAGED void Release() + { + ReleaseShared(); + } +}; + +class KRTL_CLASS KLockThisExclusive : protected KLockHolder +{ +public: + + PAGED KLockThisExclusive(KPushLockBase &lock) : KLockHolder(lock) + { + AcquireExclusive(); + } + + PAGED void Acquire() + { + AcquireExclusive(); + } + + PAGED void Release() + { + ReleaseExclusive(); + } +}; + + diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h new file mode 100644 index 00000000..631c08bf --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h @@ -0,0 +1,84 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. + +#pragma once + +#include <wil/common.h> + +#ifdef _KERNEL_MODE + +// nullptr_t is normally automatically defined by the CRT headers, but it +// doesn't get included by kernel code. +namespace std { typedef decltype(__nullptr) nullptr_t; } +using ::std::nullptr_t; + +// The stddef.h used for kernel code has the old offsetof macro. +// Let's use the new one instead. +#undef offsetof +#define offsetof(s,m) __builtin_offsetof(s,m) + +#endif // _KERNEL_MODE + +#define BEGIN_MACRO do { +#define END_MACRO } while (0) + +#ifdef _KERNEL_MODE +#define CODE_SEG(segment) __declspec(code_seg(segment)) +#else +#define CODE_SEG(segment) +#endif + +#ifndef KRTL_PAGE_SEGMENT +# define KRTL_PAGE_SEGMENT "PAGE" +#endif +#ifndef KRTL_INIT_SEGMENT +# define KRTL_INIT_SEGMENT "INIT" +#endif +#ifndef KRTL_NONPAGED_SEGMENT +# define KRTL_NONPAGED_SEGMENT ".text" +#endif + +/// Use on pageable functions. +#define PAGED CODE_SEG(KRTL_PAGE_SEGMENT) _IRQL_always_function_max_(PASSIVE_LEVEL) + +/// Use on pageable functions, where you don't want the SAL IRQL annotation to say PASSIVE_LEVEL. +#define PAGEDX CODE_SEG(KRTL_PAGE_SEGMENT) + +/// Use on code in the INIT segment. (Code is discarded after DriverEntry returns.) +#define INITCODE CODE_SEG(KRTL_INIT_SEGMENT) + +/// Use on code that must always be locked in memory. +#define NONPAGED CODE_SEG(KRTL_NONPAGED_SEGMENT) _IRQL_requires_max_(DISPATCH_LEVEL) + +/// Use on code that must always be locked in memory, where you don't want SAL IRQL annotations. +#define NONPAGEDX CODE_SEG(KRTL_NONPAGED_SEGMENT) + +#ifndef _KERNEL_MODE + +#ifndef PAGED_CODE +#define PAGED_CODE() (void)0 +#endif // PAGED_CODE + +#endif // _KERNEL_MODE + +/// Use on classes or structs. Class member functions & compiler-generated code +/// will default to the PAGE segment. You can override any member function with `NONPAGED`. +#define KRTL_CLASS CODE_SEG(KRTL_PAGE_SEGMENT) __declspec(empty_bases) + +/// Use on classes or structs. Class member functions & compiler-generated code +/// will default to the NONPAGED segment. You can override any member function with `PAGED`. +#define KRTL_CLASS_DPC_ALLOC __declspec(empty_bases) + +enum CallRunMode +{ + // This call should complete synchronously on the current thread + RunSynchronous, + // This call should return immediately, and complete the operation in a background thread + RunAsynchronous, + // This call can return immediately, OR complete synchronously + // (Use this if you're running on a workitem thread already, and you + // don't mind if the callee uses your thread to do its work, but you + // can tolerate the call completing asynchronously if the callee doesn't + // need your thread.) + RunAsynchronousButOkayToBlock, +}; + diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h new file mode 100644 index 00000000..c589866c --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h @@ -0,0 +1,147 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. + +#pragma once + +#include <KMacros.h> +#include <KCriticalRegion.h> + +typedef struct _KTHREAD *PKTHREAD; +// Copyright (C) Microsoft Corporation. All rights reserved. + +class KPushLockBase +{ +public: + + KPushLockBase() = default; + KPushLockBase(KPushLockBase &) = delete; + KPushLockBase & operator=(KPushLockBase &) = delete; + + PAGED + void + KPushLockBase::AcquireShared() + { +#ifdef _KERNEL_MODE + ExAcquirePushLockShared(&m_Lock); +#else + AcquireSRWLockShared(&m_Lock); +#endif + } + + PAGED + void + KPushLockBase::ReleaseShared() + { +#ifdef _KERNEL_MODE + ExReleasePushLockShared(&m_Lock); +#else + ReleaseSRWLockShared(&m_Lock); +#endif + } + + PAGED + void + KPushLockBase::AcquireExclusive() + { +#ifdef _KERNEL_MODE + ExAcquirePushLockExclusive(&m_Lock); +#if DBG + m_ExclusiveOwner = KeGetCurrentThread(); +#endif +#else + AcquireSRWLockExclusive(&m_Lock); +#endif + + } + + PAGED + void + KPushLockBase::ReleaseExclusive() + { +#if DBG + m_ExclusiveOwner = nullptr; +#endif +#ifdef _KERNEL_MODE + ExReleasePushLockExclusive(&m_Lock); +#else + ReleaseSRWLockExclusive(&m_Lock); +#endif + } + + PAGED + void + KPushLockBase::AssertLockHeld() + { +#ifdef _KERNEL_MODE + WIN_ASSERT(m_ExclusiveOwner == KeGetCurrentThread()); +#endif + } + + PAGED + void + KPushLockBase::AssertLockNotHeld() + { +#if DBG && defined(_KERNEL_MODE) + WIN_ASSERT(m_ExclusiveOwner != KeGetCurrentThread()); +#endif + } + +protected: + + PAGED + void + KPushLockBase::InitializeInner() + { +#ifdef _KERNEL_MODE + ExInitializePushLock(&m_Lock); +#else + InitializeSRWLock(&m_Lock); +#endif +#if DBG + m_ExclusiveOwner = nullptr; +#endif + } + +private: + +#ifdef _KERNEL_MODE + EX_PUSH_LOCK m_Lock; +#else + SRWLOCK m_Lock; +#endif + +#if DBG + PKTHREAD m_ExclusiveOwner; +#endif +}; + +class KPushLock : public KPushLockBase +{ +public: + + PAGED + KPushLock::KPushLock() noexcept + { + InitializeInner(); + } + + + + PAGED + KPushLock::~KPushLock() + { + AssertLockNotHeld(); + } +}; + +class KPushLockManualConstruct : public KPushLockBase +{ +public: + + PAGED + void + KPushLockManualConstruct::Initialize() + { + InitializeInner(); + } +}; + diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h new file mode 100644 index 00000000..124f416e --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h @@ -0,0 +1,172 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. +#pragma once + +#include <KMacros.h> + +#if _KERNEL_MODE +typedef wistd::integral_constant<EVENT_TYPE, SynchronizationEvent> auto_reset_event_t; +#else +typedef wistd::integral_constant<bool, false> auto_reset_event_t; +#endif + +#if _KERNEL_MODE +typedef wistd::integral_constant<EVENT_TYPE, NotificationEvent> manual_reset_event_t; +#else +typedef wistd::integral_constant<bool, true> manual_reset_event_t; +#endif + +#define WIN_VERIFY NT_VERIFY +#define WIN_ASSERT NT_ASSERT + +template<typename TEventType> +class KWaitEventBase +{ +public: + + KWaitEventBase() = default; + +#ifdef _KERNEL_MODE + NONPAGED ~KWaitEventBase() = default; +#else + NONPAGED ~KWaitEventBase() + { + WIN_ASSERT(m_event == nullptr); + } +#endif + + KWaitEventBase(KWaitEventBase &) = delete; + KWaitEventBase(KWaitEventBase &&) = delete; + KWaitEventBase & operator=(KWaitEventBase &) = delete; + KWaitEventBase & operator=(KWaitEventBase &&) = delete; + + _IRQL_requires_max_(DISPATCH_LEVEL) void Set() + { +#if _KERNEL_MODE + KeSetEvent(&m_event, 0, false); +#else + WIN_VERIFY(SetEvent(m_event)); +#endif + } + + _IRQL_requires_max_(DISPATCH_LEVEL) void Clear() + { +#if _KERNEL_MODE + KeClearEvent(&m_event); +#else + WIN_VERIFY(ResetEvent(m_event)); +#endif + } + + PAGED void Wait() + { +#if _KERNEL_MODE + // Not used at runtime, but might be useful during debugging + volatile LARGE_INTEGER SystemTime; + KeQuerySystemTime(const_cast<LARGE_INTEGER*>(&SystemTime)); + + NTSTATUS NtStatus = KeWaitForSingleObject( + &m_event, Executive, KernelMode, FALSE, nullptr); + NT_VERIFY(NtStatus == STATUS_SUCCESS); +#else + ULONG r = WaitForSingleObject(m_event, INFINITE); + WIN_VERIFY(r == NO_ERROR); +#endif + } + + PAGED bool Test() + { +#if _KERNEL_MODE + return !!KeReadStateEvent(&m_event); +#else + ULONG r = WaitForSingleObject(m_event, 0); + WIN_VERIFY(r == WAIT_TIMEOUT || r == WAIT_OBJECT_0); + return (r == WAIT_OBJECT_0); +#endif + } + + NONPAGED bool TestNP() + { +#if _KERNEL_MODE + return !!KeReadStateEvent(&m_event); +#else + ULONG r = WaitForSingleObject(m_event, 0); + WIN_VERIFY(r == WAIT_TIMEOUT || r == WAIT_OBJECT_0); + return (r == WAIT_OBJECT_0); +#endif + } + +protected: + + PAGED void InitializeBase() + { +#if _KERNEL_MODE + KeInitializeEvent(&m_event, TEventType(), FALSE); +#else + m_event = CreateEventW(nullptr, TEventType(), false, nullptr); + WIN_VERIFY(m_event); +#endif + } + + NONPAGED void CleanupBase() + { +#ifndef _KERNEL_MODE + CloseHandle(m_event); + m_event = nullptr; +#endif + } + +private: + +#if _KERNEL_MODE + KEVENT m_event; +#else + HANDLE m_event; +#endif + +}; + +class KWaitEvent : public KWaitEventBase<manual_reset_event_t> +{ +public: + + PAGED KWaitEvent() noexcept + { + InitializeBase(); + } + + NONPAGED ~KWaitEvent() + { + CleanupBase(); + } +}; + +class KWaitEventManualConstruct : public KWaitEventBase<manual_reset_event_t> +{ +public: + + PAGED void Initialize() + { + InitializeBase(); + } + + PAGED void Cleanup() + { + CleanupBase(); + } +}; + + +class KAutoEvent : public KWaitEventBase<auto_reset_event_t> +{ +public: + + PAGED KAutoEvent() noexcept + { + InitializeBase(); + } + + NONPAGED ~KAutoEvent() + { + CleanupBase(); + } +}; diff --git a/network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp b/network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp new file mode 100644 index 00000000..6eed638c --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +#include "pch.hpp" +#include "netvadapter.h" +#include "rxqueue.h" +#include "memory.h" + +static +void +CheckForWakeFrame( + NetvRxQueue * rx +) +{ + NET_RING_FRAGMENT_ITERATOR fi = NetRingGetAllFragments(rx->m_rings); + + if (! NetFragmentIteratorHasAny(&fi)) + { + return; + } + + auto *fragment = NetFragmentIteratorGetFragment(&fi); + auto *rxVirtualAddress = NetExtensionGetFragmentVirtualAddress( + &rx->VirtualAddressExtension, + NetFragmentIteratorGetIndex(&fi)); + + auto *fragmentBuffer = reinterpret_cast<unsigned char *>(rxVirtualAddress->VirtualAddress) + fragment->Offset; + + fragment->ValidLength = EnlCopyWakeFrame( + NetvEnlMLink[rx->m_adapter.EnlIndex].LinkHandle[0], + fragmentBuffer, + fragment->Capacity); + + // If there was a pending wake frame mark this fragment as complete, the normal advance code will get to it + fragment->Scratch = fragment->ValidLength > 0 ? 1 : 0; + + rx->CheckedWakeFrame = true; +} + + +NetvRxQueue::NetvRxQueue( + NETPACKETQUEUE Handle, + NetvAdapter & Adapter +) + : NetvQueue{Handle, Adapter, NetRxQueueGetRingCollection(Handle)} +{ + NET_EXTENSION_QUERY extension; + + NET_EXTENSION_QUERY_INIT( + &extension, + NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_NAME, + NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_VERSION_1, + NetExtensionTypeFragment); + + NetRxQueueGetExtension(m_handle, &extension, &VirtualAddressExtension); + + NET_EXTENSION_QUERY_INIT( + &extension, + NET_PACKET_EXTENSION_RSC_NAME, + NET_PACKET_EXTENSION_RSC_VERSION_2, + NetExtensionTypePacket); + + NetRxQueueGetExtension(m_handle, &extension, &UdpRscExtension); + + NET_EXTENSION_QUERY_INIT( + &extension, + NET_PACKET_EXTENSION_CHECKSUM_NAME, + NET_PACKET_EXTENSION_CHECKSUM_VERSION_1, + NetExtensionTypePacket); + + NetRxQueueGetExtension(m_handle, &extension, &RxXSumExtension); + + EnlQueueHandle = EnlCreateQueue(Handle, RX); +} + +_Use_decl_annotations_ +void +NetvRxQueue::Destroy( + void +) +{ + EnlDestroyQueue(EnlQueueHandle, RX); +} + +void +NetvRxQueue::Start( + void +) +{ + auto link = NetvEnlMLink[m_adapter.EnlIndex].LinkHandle[0]; + auto port = &link->Ports[m_adapter.EnlPortIndex]; + auto queue = &port->RxQueue[0]; + + WDFVERIFY(queue->State == Stopped); + + queue->QueueNext = queue->QueueEnd = 0U; + + EnlIndicateQueueState(EnlQueueHandle, Started); +} + +void +NetvRxQueue::Stop( + void +) +{ + EnlIndicateQueueState(EnlQueueHandle, Stopped); +} + +_Use_decl_annotations_ +void +NetvRxQueue::Advance( + void +) +{ + auto fr = GetFragmentRing(); + NET_RING_PACKET_ITERATOR pi = NetRingGetAllPackets(m_rings); + NET_RING_FRAGMENT_ITERATOR fi = NetRingGetAllFragments(m_rings); + + // Ideally this would run in EvtQueueStart, but at that point the receive buffers are not + // attached to the fragment yet + if (! CheckedWakeFrame) + { + CheckForWakeFrame(this); + } + + // Move begin index forward for all fragments with Scratch == 1, thus returning them to the OS since we're done processing them. + for (; NetFragmentIteratorHasAny(&fi) && NetPacketIteratorHasAny(&pi); NetPacketIteratorAdvance(&pi), NetFragmentIteratorAdvance(&fi)) + { + NET_FRAGMENT const * fragment = NetFragmentIteratorGetFragment(&fi); + if (! fragment->Scratch) + { + break; + } + } + + NetFragmentIteratorSet(&fi); + NetPacketIteratorSet(&pi); + EnlRingDoorBell(EnlQueueHandle, fr->EndIndex); +} + +_Use_decl_annotations_ +void +NetvRxQueue::Cancel( + void +) +{ + CancelRxPackets(m_rings); +} + +_Use_decl_annotations_ +void +NetvRxQueue::SetNotify( + bool NotificationEnabled +) +{ + EnlArmInterrupt(EnlQueueHandle, NotificationEnabled); +} diff --git a/network/netadaptercx/netvadapterlibrary/code/rxqueue.h b/network/netadaptercx/netvadapterlibrary/code/rxqueue.h new file mode 100644 index 00000000..cf208def --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/rxqueue.h @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved +#pragma once +#include "queue.h" + +class NetvRxQueue final + : public NetvQueue +{ + +public: + + NetvRxQueue( + NETPACKETQUEUE Handle, + NetvAdapter & Adapter + ); + + void + Destroy( + void + ); + + void + Start( + void + ); + + void + Stop( + void + ); + + void + Advance( + void + ); + + void + Cancel( + void + ); + + void + SetNotify( + bool Enable + ); + + ENLP_QUEUE * EnlQueueHandle; + + NET_EXTENSION VirtualAddressExtension; + NET_EXTENSION UdpRscExtension; + NET_EXTENSION RxXSumExtension; + NET_EXTENSION NetMemoryExtension; + NET_EXTENSION NetMemoryReturnContextExtensionIn; + + bool CheckedWakeFrame = false; +}; + +WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(NetvRxQueue, NetvRxQueueGetContext); diff --git a/network/netadaptercx/netvadapterlibrary/code/trace.h b/network/netadaptercx/netvadapterlibrary/code/trace.h new file mode 100644 index 00000000..dc2e28e4 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/trace.h @@ -0,0 +1,96 @@ +// Copyright (C) Microsoft Corporation. All rights reserved. + +#pragma once + +#include <initguid.h> + +// 5EC34F87-7705-4B5B-B5F6-1F3E4665A31E +#define WPP_CONTROL_GUIDS \ + WPP_DEFINE_CONTROL_GUID( \ + NetvadapterTraceGuid, \ + (5EC34F87,7705,4B5B,B5F6,1F3E4665A31E), \ + WPP_DEFINE_BIT(FLAG_DRIVER) \ + ) + +#define WPP_FLAG_LEVEL_LOGGER(flag, level) \ + WPP_LEVEL_LOGGER(flag) + +#define WPP_FLAG_LEVEL_ENABLED(flag, level) \ + (WPP_LEVEL_ENABLED(flag) && WPP_CONTROL(WPP_BIT_ ## flag).Level >= level) + +#define WPP_LEVEL_FLAGS_LOGGER(lvl,flags) \ + WPP_LEVEL_LOGGER(flags) + +#define WPP_LEVEL_FLAGS_ENABLED(lvl, flags) \ + (WPP_LEVEL_ENABLED(flags) && WPP_CONTROL(WPP_BIT_ ## flags).Level >= lvl) + +// begin_wpp config +// USEPREFIX (LogInformation, "%!FUNC! ->"); +// LogVerbose{LEVEL=TRACE_LEVEL_VERBOSE}(FLAGS, MSG, ...); +// end_wpp + +// begin_wpp config +// USEPREFIX (LogInformation, "%!FUNC! ->"); +// LogWarning{LEVEL=TRACE_LEVEL_WARNING}(FLAGS, MSG, ...); +// end_wpp + +// begin_wpp config +// USEPREFIX (LogInformation, "%!FUNC! ->"); +// LogError{LEVEL=TRACE_LEVEL_ERROR}(FLAGS, MSG, ...); +// end_wpp + +// begin_wpp config +// USEPREFIX (LogInformation, "%!FUNC! ->"); +// LogInformation{LEVEL=TRACE_LEVEL_INFORMATION}(FLAGS, MSG, ...); +// end_wpp + +// +// WPP orders static parameters before dynamic parameters. To support the Trace function +// defined below which sets FLAGS=MYDRIVER_ALL_INFO, a custom macro must be defined to +// reorder the arguments to what the .tpl configuration file expects. +// +#define WPP_RECORDER_FLAGS_LEVEL_ARGS(flags, lvl) \ + WPP_RECORDER_LEVEL_FLAGS_ARGS(lvl, flags) + +#define WPP_RECORDER_FLAGS_LEVEL_FILTER(flags, lvl) \ + WPP_RECORDER_LEVEL_FLAGS_FILTER(lvl, flags) + +// begin_wpp config +// USEPREFIX (RETURN_IF_NOT_STATUS_SUCCESS, "%!STATUS! %!FUNC! ->%!s!", nt__wpp, #NTSTATUS); +// FUNC RETURN_IF_NOT_STATUS_SUCCESS{FLAG=FLAG_DRIVER,LEVEL=TRACE_LEVEL_ERROR}(NTSTATUS); +// end_wpp + +#define WPP_FLAG_LEVEL_NTSTATUS_PRE(flag, level, ntstatus) do { NTSTATUS nt__wpp = (ntstatus); if (STATUS_SUCCESS != nt__wpp) { +#define WPP_FLAG_LEVEL_NTSTATUS_POST(flag, level, ntstatus); return nt__wpp; } } while (0) +#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_FILTER(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag) +#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_ARGS(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag) + +// begin_wpp config +// USEPREFIX (RETURN_NTSTATUS_IF, "%!STATUS! %!FUNC! ->%!s!", nt__wpp, #CONDITION); +// FUNC RETURN_NTSTATUS_IF{FLAG=FLAG_DRIVER,LEVEL=TRACE_LEVEL_ERROR}(NTSTATUS, CONDITION); +// end_wpp + +#define WPP_FLAG_LEVEL_NTSTATUS_CONDITION_PRE(flag, level, ntstatus, condition) if (condition) { NTSTATUS nt__wpp = (ntstatus); +#define WPP_FLAG_LEVEL_NTSTATUS_CONDITION_POST(flag, level, ntstatus, condition); return nt__wpp; } +#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_CONDITION_FILTER(flag, level, ntstatus, condition) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag) +#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_CONDITION_ARGS(flag, level, ntstatus, condition) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag) + +// begin_wpp config +// USEPREFIX (RETURN_FAILED_NTSTATUS_MSG, "%!STATUS! %!FUNC! ->", nt__wpp); +// FUNC RETURN_FAILED_NTSTATUS_MSG{FLAG=FLAG_DRIVER,FAILEDLEVEL=TRACE_LEVEL_ERROR}(NTSTATUS, MSG, ...); +// end_wpp + +#define WPP_FLAG_FAILEDLEVEL_NTSTATUS_PRE(flag, level, ntstatus); do { NTSTATUS nt__wpp = (ntstatus); +#define WPP_FLAG_FAILEDLEVEL_NTSTATUS_POST(flag, level, ntstatus); return nt__wpp; } while (0) +#define WPP_RECORDER_FLAG_FAILEDLEVEL_NTSTATUS_FILTER(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag) +#define WPP_RECORDER_FLAG_FAILEDLEVEL_NTSTATUS_ARGS(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag) + +// begin_wpp config +// USEPREFIX (RETURN_STATUS_SUCCESS, "%!STATUS! %!FUNC!", STATUS_SUCCESS); +// FUNC RETURN_STATUS_SUCCESS{FLAG=FLAG_DRIVER,SUCCESSLEVEL=TRACE_LEVEL_INFORMATION,NTSTATUS=STATUS_SUCCESS}(); +// end_wpp + +#define WPP_FLAG_SUCCESSLEVEL_NTSTATUS_POST(flag, level, ntstatus); return (ntstatus); +#define WPP_RECORDER_FLAG_SUCCESSLEVEL_NTSTATUS_FILTER(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag) +#define WPP_RECORDER_FLAG_SUCCESSLEVEL_NTSTATUS_ARGS(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag) + diff --git a/network/netadaptercx/netvadapterlibrary/code/txqueue.cpp b/network/netadaptercx/netvadapterlibrary/code/txqueue.cpp new file mode 100644 index 00000000..b67692b5 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/txqueue.cpp @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +#include "pch.hpp" +#include "netvadapter.h" +#include "txqueue.h" + +NetvTxQueue::NetvTxQueue( + NETPACKETQUEUE Handle, + NetvAdapter & Adapter +) noexcept + : NetvQueue{Handle, Adapter, NetTxQueueGetRingCollection(Handle)} +{ + NET_EXTENSION_QUERY extension; + NET_EXTENSION_QUERY_INIT( + &extension, + NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_NAME, + NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_VERSION_1, + NetExtensionTypeFragment); + + NetTxQueueGetExtension(m_handle, &extension, &VirtualAddressExtension); + + NET_EXTENSION_QUERY_INIT( + &extension, + NET_PACKET_EXTENSION_GSO_NAME, + NET_PACKET_EXTENSION_GSO_VERSION_1, + NetExtensionTypePacket); + + NetTxQueueGetExtension(m_handle, &extension, &UsoExtension); + + EnlQueueHandle = EnlCreateQueue(Handle, TX); +} + +_Use_decl_annotations_ +void +NetvTxQueue::Destroy( + void +) +{ + EnlDestroyQueue(EnlQueueHandle, TX); +} + +void +NetvTxQueue::Start( + void +) +{ + auto link = NetvEnlMLink[m_adapter.EnlIndex].LinkHandle[0]; + auto port = &link->Ports[m_adapter.EnlPortIndex]; + auto queue = &port->TxQueue[0]; + + NT_ASSERT(queue->State == Stopped); + + queue->QueueNext = queue->QueueEnd = 0U; + + EnlIndicateQueueState(EnlQueueHandle, Started); +} + +void +NetvTxQueue::Stop( + void +) +{ + EnlIndicateQueueState(EnlQueueHandle, Stopped); +} + +_Use_decl_annotations_ +void +NetvTxQueue::Advance( + void +) +{ + auto pr = GetPacketRing(); + auto pi = NetRingGetAllPackets(m_rings); + + // drain Tx packets + for (; NetPacketIteratorHasAny(&pi); NetPacketIteratorAdvance(&pi)) + { + auto packet = NetPacketIteratorGetPacket(&pi); + if (! packet->Scratch) + { + break; + } + + auto fi = NetPacketIteratorGetFragments(&pi); + for (; NetFragmentIteratorHasAny(&fi); NetFragmentIteratorAdvance(&fi)) + { + continue; + } + + m_rings->Rings[NetRingTypeFragment]->BeginIndex = + NetFragmentIteratorGetIndex(&fi); + } + + NetPacketIteratorSet(&pi); + + // post Tx packets + EnlRingDoorBell(EnlQueueHandle, pr->EndIndex); +} + +_Use_decl_annotations_ +void +NetvTxQueue::Cancel( + void +) +{ + auto ringBuffer = GetPacketRing(); + + EnlRingDoorBell(EnlQueueHandle, ringBuffer->EndIndex); +} + +_Use_decl_annotations_ +void +NetvTxQueue::SetNotify( + bool NotificationEnabled +) +{ + EnlArmInterrupt(EnlQueueHandle, NotificationEnabled); +} diff --git a/network/netadaptercx/netvadapterlibrary/code/txqueue.h b/network/netadaptercx/netvadapterlibrary/code/txqueue.h new file mode 100644 index 00000000..425a38e9 --- /dev/null +++ b/network/netadaptercx/netvadapterlibrary/code/txqueue.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved + +#include "queue.h" + +class NetvTxQueue final + : public NetvQueue +{ + +public: + + NetvTxQueue( + NETPACKETQUEUE Handle, + NetvAdapter & Adapter + ) noexcept; + + void + Destroy( + void + ); + + void + Start( + void + ); + + void + Stop( + void + ); + + void + Advance( + void + ); + + void + Cancel( + void + ); + + void + SetNotify( + bool Enable + ); + + ENLP_QUEUE * EnlQueueHandle; + NET_EXTENSION VirtualAddressExtension; + NET_EXTENSION UsoExtension; +}; + +WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(NetvTxQueue, NetvTxQueueGetContext); |
