summaryrefslogtreecommitdiff
path: root/network/netadaptercx/netvadapterlibrary/code
diff options
context:
space:
mode:
author5an7y-Microsoft <[email protected]>2026-06-19 13:23:44 -0700
committerGitHub <[email protected]>2026-06-19 13:23:44 -0700
commit26741c75857c7df2fd95efe632f8f5f70b71fdbb (patch)
tree58d74336b33f09a11d9427f8eba76f7fa84d9d47 /network/netadaptercx/netvadapterlibrary/code
parente99ae832b48b245404f9bd750af4864247b061e8 (diff)
parent8315a48cc6aac0c02d2218b0147457a01de4f885 (diff)
Merge pull request #1394 from youyanggit/WifiihvsampleDataPath
Add End-to-End NetAdapterCx/WiFiCx Sample Drivers and OEM Communication Examples
Diffstat (limited to 'network/netadaptercx/netvadapterlibrary/code')
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/adapter.cpp595
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/configuration.cpp102
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/configuration.h8
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/enl.cpp1001
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/enl.h264
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp136
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/enlthreads.h119
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/memory.cpp11
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/memory.h4
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h85
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h281
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h268
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/pch.hpp34
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/queue.cpp34
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/queue.h41
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h67
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h114
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h84
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h147
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h172
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp156
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/rxqueue.h57
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/trace.h96
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/txqueue.cpp118
-rw-r--r--network/netadaptercx/netvadapterlibrary/code/txqueue.h51
25 files changed, 4045 insertions, 0 deletions
diff --git a/network/netadaptercx/netvadapterlibrary/code/adapter.cpp b/network/netadaptercx/netvadapterlibrary/code/adapter.cpp
new file mode 100644
index 00000000..880f970d
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/adapter.cpp
@@ -0,0 +1,595 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+#include "pch.hpp"
+
+#include <new.h>
+#ifdef _KERNEL_MODE
+#include <xfilter.h>
+#else
+#include "net/umxfilter.h" // Copied from Km XFilter.h, NETCX please move this xfilter.h into shared location
+#endif
+#include "netvadapter.h"
+#include "rxqueue.h"
+#include "txqueue.h"
+#include "configuration.h"
+#include "trace.h"
+#include "memory.h"
+
+#include "adapter.tmh"
+
+UCHAR NetvMacAddressBase[MAC_ADDR_LEN] = { 0x22, 0x22, 0x22, 0x22, 0x00, 0x00 };
+const ULONG GSO_MAX_OFFLOAD_SIZE = 0xffff;
+const ULONG GSO_MIN_SEGMENT_COUNT = 2;
+
+/*
+ * increasing beyond 1Gbps results in intermittent failure of
+ * netvadapter start due to buffer allocation failures in
+ * netadaptercx when running in nebula.
+ * tracked as bug 50671552 (if it doesn't get archived)
+ */
+static auto constexpr MAX_LINK_SPEED{1'000'000'000ull};
+
+void
+NetvEnlInterruptRoutine(
+ _Inout_ PVOID PortContext,
+ _In_ bool Tx
+)
+{
+ NETPACKETQUEUE queue = (NETPACKETQUEUE)PortContext;
+
+ if (Tx) // Tx
+ {
+ NetTxQueueNotifyMoreCompletedPacketsAvailable(queue);
+ }
+ else // Rx
+ {
+ NetRxQueueNotifyMoreReceivedPacketsAvailable(queue);
+ }
+}
+
+static
+EVT_PACKET_QUEUE_START
+ EvtTxQueueStart;
+
+static
+EVT_PACKET_QUEUE_STOP
+ EvtTxQueueStop;
+
+static
+EVT_PACKET_QUEUE_ADVANCE
+ EvtTxQueueAdvance;
+
+static
+EVT_PACKET_QUEUE_CANCEL
+ EvtTxQueueCancel;
+
+static
+EVT_PACKET_QUEUE_SET_NOTIFICATION_ENABLED
+ EvtTxQueueSetNotify;
+
+static
+EVT_PACKET_QUEUE_START
+ EvtRxQueueStart;
+
+static
+EVT_PACKET_QUEUE_STOP
+ EvtRxQueueStop;
+
+static
+EVT_PACKET_QUEUE_ADVANCE
+ EvtRxQueueAdvance;
+
+static
+EVT_PACKET_QUEUE_CANCEL
+ EvtRxQueueCancel;
+
+static
+EVT_PACKET_QUEUE_SET_NOTIFICATION_ENABLED
+ EvtRxQueueSetNotify;
+
+NetvAdapter::NetvAdapter(
+ NETADAPTER Handle,
+ WDFDEVICE Device
+) noexcept
+ : m_handle(Handle)
+ , m_device(Device)
+{
+}
+
+NTSTATUS
+NetvAdapter::Initialize(
+ void
+)
+{
+ RETURN_IF_NOT_STATUS_SUCCESS(
+ NetvAdapterReadConfiguration(this, m_device));
+
+ RETURN_IF_NOT_STATUS_SUCCESS(NetvAdapterReadAddress());
+
+ SetLinkState();
+
+ NTSTATUS status = STATUS_SUCCESS;
+
+ // Create ENL
+ EnlPortCreated = FALSE;
+
+ if (NetvEnlMLink[EnlIndex].LinkCount == 0)
+ {
+ RETURN_IF_NOT_STATUS_SUCCESS(
+ EnlMCreateLink(
+ LinkCount,
+ LinkProcIndex,
+ LinkPoll,
+ &NetvEnlMLink[EnlIndex]));
+ }
+
+ RETURN_NTSTATUS_IF(STATUS_INVALID_ADDRESS,
+ EnlMIsPortActive(&NetvEnlMLink[EnlIndex], EnlPortIndex));
+
+ status = EnlMActivateLinkPort(
+ &NetvEnlMLink[EnlIndex],
+ EnlPortIndex,
+ NetvEnlInterruptRoutine,
+ this);
+
+ // what is the virtue of doing this?
+ if (NT_SUCCESS(status))
+ {
+ EnlPortCreated = TRUE;
+ }
+
+ RETURN_STATUS_SUCCESS();
+}
+
+void
+NetvAdapter::Destroy(
+ void
+)
+{
+ if (EnlPortCreated)
+ {
+ EnlMDeactivateLinkPort(&NetvEnlMLink[EnlIndex], EnlPortIndex);
+ EnlPortCreated = FALSE;
+ }
+
+ if (EnlIndexValid && !EnlIsLinkActive(NetvEnlMLink[EnlIndex].LinkHandle[0]))
+ {
+ EnlMCloseLink(&NetvEnlMLink[EnlIndex]);
+ }
+}
+
+_Use_decl_annotations_
+NTSTATUS
+NetvAdapter::CreateRxQueue(
+ NETRXQUEUE_INIT * NetRxQueueInit
+ )
+{
+ WDF_OBJECT_ATTRIBUTES rxAttributes;
+ WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&rxAttributes, NetvRxQueue);
+ rxAttributes.EvtDestroyCallback = [](WDFOBJECT Handle) {
+ NetvRxQueueGetContext(static_cast<NETPACKETQUEUE>(Handle))->Destroy();
+ };
+
+ NET_PACKET_QUEUE_CONFIG rxConfig;
+ NET_PACKET_QUEUE_CONFIG_INIT(
+ &rxConfig,
+ EvtRxQueueAdvance,
+ EvtRxQueueSetNotify,
+ EvtRxQueueCancel);
+ rxConfig.EvtStart = EvtRxQueueStart;
+ rxConfig.EvtStop = EvtRxQueueStop;
+
+ NETPACKETQUEUE rxQueue;
+ RETURN_IF_NOT_STATUS_SUCCESS(NetRxQueueCreate(
+ NetRxQueueInit,
+ &rxAttributes,
+ &rxConfig,
+ &rxQueue));
+
+ new (NetvRxQueueGetContext(rxQueue)) NetvRxQueue(rxQueue, *this);
+
+ RETURN_STATUS_SUCCESS();
+}
+
+_Use_decl_annotations_
+NTSTATUS
+NetvAdapter::CreateTxQueue(
+ NETTXQUEUE_INIT * NetTxQueueInit
+ )
+{
+ WDF_OBJECT_ATTRIBUTES txAttributes;
+ WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&txAttributes, NetvTxQueue);
+ txAttributes.EvtDestroyCallback = [](WDFOBJECT Handle) {
+ NetvTxQueueGetContext(static_cast<NETPACKETQUEUE>(Handle))->Destroy();
+ };
+
+ NET_PACKET_QUEUE_CONFIG txConfig;
+ NET_PACKET_QUEUE_CONFIG_INIT(
+ &txConfig,
+ EvtTxQueueAdvance,
+ EvtTxQueueSetNotify,
+ EvtTxQueueCancel);
+ txConfig.EvtStart = EvtTxQueueStart;
+ txConfig.EvtStop = EvtTxQueueStop;
+
+ NETPACKETQUEUE txQueue;
+ RETURN_IF_NOT_STATUS_SUCCESS(NetTxQueueCreate(
+ NetTxQueueInit,
+ &txAttributes,
+ &txConfig,
+ &txQueue));
+
+ new (NetvTxQueueGetContext(txQueue)) NetvTxQueue(txQueue, *this);
+
+ RETURN_STATUS_SUCCESS();
+}
+
+void NetvAdapter::SetPdoWakeSignalCallback(_In_ EVT_PDO_WAKE_SIGNAL* evtPdoWakeSignal, _In_ void* context)
+{
+ EnlSetPdoWakeSignalCallback(NetvEnlMLink[EnlIndex].LinkHandle[0], evtPdoWakeSignal, context);
+}
+
+_Use_decl_annotations_
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void NetvAdapter::ArmWakeFromS0(void)
+{
+ if (EnlPortCreated)
+ {
+ ENLP_LINK* enLinkHandle = NetvEnlMLink[EnlIndex].LinkHandle[0];
+ EnlArmWake(enLinkHandle);
+ }
+}
+
+_Use_decl_annotations_
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void NetvAdapter::DisarmWakeFromS0(void)
+{
+ if (EnlPortCreated)
+ {
+ ENLP_LINK* enLinkHandle = NetvEnlMLink[EnlIndex].LinkHandle[0];
+ EnlDisarmWake(enLinkHandle);
+ }
+}
+
+_Use_decl_annotations_
+void
+NetvAdapter::SetLinkState(
+ void
+) const
+{
+ NET_ADAPTER_AUTO_NEGOTIATION_FLAGS autoNegotiationFlags{NetAdapterAutoNegotiationFlagNone};
+ if (LinkAutoNeg)
+ {
+ autoNegotiationFlags |=
+ NetAdapterAutoNegotiationFlagXmitLinkSpeedAutoNegotiated |
+ NetAdapterAutoNegotiationFlagRcvLinkSpeedautoNegotiated |
+ NetAdapterAutoNegotiationFlagDuplexAutoNegotiated;
+ }
+ if (FlowControl != NetvFlowControlDisabled)
+ {
+ autoNegotiationFlags |=
+ NetAdapterAutoNegotiationFlagPauseFunctionsAutoNegotiated;
+ }
+
+ NET_ADAPTER_PAUSE_FUNCTION_TYPE pauseFunctions{NetAdapterPauseFunctionTypeUnknown};
+ switch (FlowControl)
+ {
+ case NetvFlowControlDisabled:
+ pauseFunctions = NetAdapterPauseFunctionTypeUnsupported;
+ break;
+ case NetvFlowControlRxEnabled:
+ pauseFunctions = NetAdapterPauseFunctionTypeReceiveOnly;
+ break;
+ case NetvFlowControlTxEnabled:
+ pauseFunctions = NetAdapterPauseFunctionTypeSendOnly;
+ break;
+ case NetvFlowControlTxRxEnabled:
+ pauseFunctions = NetAdapterPauseFunctionTypeSendAndReceive;
+ break;
+ }
+
+ NET_ADAPTER_LINK_STATE linkState;
+ NET_ADAPTER_LINK_STATE_INIT(
+ &linkState,
+ MAX_LINK_SPEED,
+ MediaConnectStateConnected,
+ MediaDuplexStateFull,
+ pauseFunctions,
+ autoNegotiationFlags);
+ NetAdapterSetLinkState(m_handle, &linkState);
+}
+
+static
+void
+EvtSetReceiveFilter(
+ _In_ NETADAPTER NetAdapter,
+ _In_ NETRECEIVEFILTER Handle
+ )
+{
+ NetvAdapter* adapter = NetvAdapterGetContextFromWDFObject(NetAdapter);
+
+ adapter->PacketFilter = NetReceiveFilterGetPacketFilter(Handle);
+
+ adapter->NumMulticastAddresses = (ULONG)NetReceiveFilterGetMulticastAddressCount(Handle);
+
+ RtlZeroMemory(adapter->MulticastAddressList,
+ sizeof(NET_ADAPTER_LINK_LAYER_ADDRESS) * MAX_MULTICAST_LIST_SIZE);
+
+ if (adapter->NumMulticastAddresses != 0U)
+ {
+ NET_ADAPTER_LINK_LAYER_ADDRESS const * MulticastAddressList = NetReceiveFilterGetMulticastAddressList(Handle);
+ RtlCopyMemory(adapter->MulticastAddressList,
+ MulticastAddressList,
+ sizeof(NET_ADAPTER_LINK_LAYER_ADDRESS) * adapter->NumMulticastAddresses);
+ }
+}
+
+static
+_IRQL_requires_same_
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+NTAPI
+EvtNetAdapterOffloadSetRxXSum(
+ _In_ NETADAPTER Adapter,
+ _In_ NETOFFLOAD Offload
+ )
+{
+ UNREFERENCED_PARAMETER((Adapter, Offload));
+ ASSERT(NetOffloadIsRxChecksumIPv4Enabled(Offload));
+ ASSERT(NetOffloadIsRxChecksumUdpEnabled(Offload));
+}
+
+static
+_IRQL_requires_same_
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+NTAPI
+EvtNetAdapterOffloadSetGso(
+ _In_ NETADAPTER Adapter,
+ _In_ NETOFFLOAD Offload
+ )
+{
+ auto adapter = NetvAdapterGetContextFromWDFObject(Adapter);
+ if (adapter->UsoEnabled)
+ {
+ //
+ // Since netvadapter only converts USO sends into URO receives,
+ // both must be enabled together, or disabled together.
+ // The order of callbacks is nondeterministic, so this can't assert
+ // that URO is enabled, since it might not have been enabled yet.
+ // This assert is so anyone using netvadapter knows that USO has
+ // been disabled after being enabled.
+ //
+ NT_FRE_ASSERTMSG("USO can't be disabled after being enabled", NetOffloadIsUsoIPv4Enabled(Offload));
+ NT_FRE_ASSERTMSG("USO can't be disabled after being enabled", NetOffloadIsUsoIPv6Enabled(Offload));
+ }
+ else
+ {
+ adapter->UsoEnabled = NetOffloadIsUsoIPv4Enabled(Offload) && NetOffloadIsUsoIPv6Enabled(Offload);
+ }
+}
+
+static
+_IRQL_requires_same_
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+NTAPI
+EvtNetAdapterOffloadSetRsc(
+ _In_ NETADAPTER Adapter,
+ _In_ NETOFFLOAD Offload
+ )
+{
+ auto adapter = NetvAdapterGetContextFromWDFObject(Adapter);
+ if (adapter->UroEnabled)
+ {
+ //
+ // Since netvadapter only converts USO sends into URO receives,
+ // both must be enabled together, or disabled together.
+ // The order of callbacks is nondeterministic, so this can't assert
+ // that USO is enabled, since it might not have been enabled yet.
+ // This assert is so anyone using netvadapter knows that URO has
+ // been disabled after being enabled.
+ // This assert is important because if URO is disabled, netvadapter
+ // will forward the USO send without fixing it up or segmenting it,
+ // and the stack may behave badly.
+ //
+ NT_FRE_ASSERTMSG("URO can't be disabled after being enabled", NetOffloadIsUdpRscEnabled(Offload));
+ }
+ else
+ {
+ adapter->UroEnabled = NetOffloadIsUdpRscEnabled(Offload);
+ }
+}
+
+static
+void
+NetvAdapterSetUsoUroOffloadCapabilities(
+ _In_ NETADAPTER Adapter
+ )
+{
+ NET_ADAPTER_OFFLOAD_GSO_CAPABILITIES gsoCapabilities;
+ NET_ADAPTER_OFFLOAD_GSO_CAPABILITIES_INIT(
+ &gsoCapabilities,
+ NetAdapterOffloadLayer3FlagIPv4NoOptions | NetAdapterOffloadLayer3FlagIPv6NoExtensions,
+ NetAdapterOffloadLayer4FlagUdp,
+ GSO_MAX_OFFLOAD_SIZE,
+ GSO_MIN_SEGMENT_COUNT,
+ EvtNetAdapterOffloadSetGso);
+
+ NET_ADAPTER_OFFLOAD_RSC_CAPABILITIES rscCapabilities;
+ NET_ADAPTER_OFFLOAD_RSC_CAPABILITIES_INIT(
+ &rscCapabilities,
+ NetAdapterOffloadLayer3FlagIPv4NoOptions | NetAdapterOffloadLayer3FlagIPv6NoExtensions,
+ NetAdapterOffloadLayer4FlagUdp,
+ EvtNetAdapterOffloadSetRsc);
+ rscCapabilities.TcpTimestampOption = FALSE;
+
+ NetAdapterOffloadSetGsoCapabilities(Adapter, &gsoCapabilities);
+ NetAdapterOffloadSetRscCapabilities(Adapter, &rscCapabilities);
+}
+
+_Use_decl_annotations_
+NTSTATUS NetvAdapter::ConfigureDataCapabilities()
+{
+
+
+ NET_ADAPTER_TX_CAPABILITIES txCapabilities;
+ NET_ADAPTER_TX_CAPABILITIES_INIT(&txCapabilities, MAX_TX_QUEUES);
+
+ NET_ADAPTER_RX_CAPABILITIES rxCapabilities;
+ NET_ADAPTER_RX_CAPABILITIES_INIT_SYSTEM_MANAGED(&rxCapabilities, MAX_RX_BUFFER_SIZE, MAX_RX_QUEUES);
+
+ NET_ADAPTER_LINK_LAYER_CAPABILITIES linkLayerCapabilities;
+ NET_ADAPTER_LINK_LAYER_CAPABILITIES_INIT(&linkLayerCapabilities, MAX_LINK_SPEED, MAX_LINK_SPEED);
+
+ NET_ADAPTER_RECEIVE_FILTER_CAPABILITIES receiveFilterCapabilities;
+ NET_ADAPTER_RECEIVE_FILTER_CAPABILITIES_INIT(&receiveFilterCapabilities, EvtSetReceiveFilter);
+ receiveFilterCapabilities.SupportedPacketFilters = NETV_SUPPORTED_FILTERS;
+ receiveFilterCapabilities.MaximumMulticastAddresses = MAX_MULTICAST_LIST_SIZE;
+
+ NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES xsumCapabilities;
+ NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES_INIT(&xsumCapabilities, EvtNetAdapterOffloadSetRxXSum);
+
+ NetAdapterSetLinkLayerCapabilities(m_handle, &linkLayerCapabilities);
+ NetAdapterSetLinkLayerMtuSize(m_handle, MTU_SIZE);
+ NetAdapterSetDataPathCapabilities(m_handle, &txCapabilities, &rxCapabilities);
+ NetAdapterSetReceiveFilterCapabilities(m_handle, &receiveFilterCapabilities);
+ NetAdapterOffloadSetRxChecksumCapabilities(m_handle, &xsumCapabilities);
+
+ NET_ADAPTER_LINK_LAYER_ADDRESS netvLinkAddress;
+ NET_ADAPTER_LINK_LAYER_ADDRESS_INIT(&netvLinkAddress, MAC_ADDR_LEN, (CONST UCHAR*)&(PermanentAddress.Address));
+
+ NetAdapterSetPermanentLinkLayerAddress(m_handle, &netvLinkAddress);
+ NetAdapterSetCurrentLinkLayerAddress(m_handle, &netvLinkAddress);
+
+ if (EnableUsoUro)
+ {
+ NetvAdapterSetUsoUroOffloadCapabilities(m_handle);
+ }
+
+ RETURN_STATUS_SUCCESS();
+}
+
+NTSTATUS
+NetvAdapter::NetvAdapterReadAddress()
+{
+ PermanentAddress.Length = MAC_ADDR_LEN;
+
+ RETURN_NTSTATUS_IF(STATUS_INVALID_ADDRESS,
+ MACLastByte < 1 ||
+ MACLastByte > MAX_ADAPTER_COUNT);
+
+ ETH_COPY_NETWORK_ADDRESS(PermanentAddress.Address, NetvMacAddressBase);
+ PermanentAddress.Address[MAC_ADDR_LEN - 1] = (UCHAR) MACLastByte;
+
+ if (ETH_IS_MULTICAST(PermanentAddress.Address) ||
+ ETH_IS_BROADCAST(PermanentAddress.Address))
+ {
+ RETURN_IF_NOT_STATUS_SUCCESS(STATUS_INVALID_ADDRESS);
+ }
+
+ RtlCopyMemory(
+ &CurrentAddress,
+ &PermanentAddress,
+ sizeof(PermanentAddress)
+ );
+
+ EnlIndex = (MACLastByte - 1) >> 1;
+ EnlPortIndex = (MACLastByte - 1) & 1;
+ EnlIndexValid = TRUE;
+
+ RETURN_STATUS_SUCCESS();
+}
+
+_Use_decl_annotations_
+void
+EvtTxQueueStart(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvTxQueueGetContext(Queue)->Start();
+}
+
+_Use_decl_annotations_
+void
+EvtTxQueueStop(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvTxQueueGetContext(Queue)->Stop();
+}
+
+_Use_decl_annotations_
+void
+EvtTxQueueAdvance(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvTxQueueGetContext(Queue)->Advance();
+}
+
+_Use_decl_annotations_
+void
+EvtTxQueueCancel(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvTxQueueGetContext(Queue)->Cancel();
+}
+
+_Use_decl_annotations_
+void
+EvtTxQueueSetNotify(
+ NETPACKETQUEUE Queue,
+ BOOLEAN Enable
+)
+{
+ NetvTxQueueGetContext(Queue)->SetNotify(Enable);
+}
+
+_Use_decl_annotations_
+void
+EvtRxQueueStart(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvRxQueueGetContext(Queue)->Start();
+}
+
+_Use_decl_annotations_
+void
+EvtRxQueueStop(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvRxQueueGetContext(Queue)->Stop();
+}
+
+
+_Use_decl_annotations_
+void
+EvtRxQueueAdvance(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvRxQueueGetContext(Queue)->Advance();
+}
+
+_Use_decl_annotations_
+void
+EvtRxQueueCancel(
+ NETPACKETQUEUE Queue
+)
+{
+ NetvRxQueueGetContext(Queue)->Cancel();
+}
+
+_Use_decl_annotations_
+void
+EvtRxQueueSetNotify(
+ NETPACKETQUEUE Queue,
+ BOOLEAN Enable
+)
+{
+ NetvRxQueueGetContext(Queue)->SetNotify(Enable);
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/configuration.cpp b/network/netadaptercx/netvadapterlibrary/code/configuration.cpp
new file mode 100644
index 00000000..7482ef84
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/configuration.cpp
@@ -0,0 +1,102 @@
+#include "pch.hpp"
+#include "netvadapter.h"
+
+#include "trace.h"
+#include "configuration.tmh"
+
+typedef struct _NETVADAPTER_ADVANCED_PROPERTY
+{
+ UNICODE_STRING RegName; // variable name text
+ UINT32 FieldOffset; // offset to NetvAdapter field
+ UINT32 FieldSize; // size (in bytes) of the field
+ UINT32 Default; // default value to use
+ UINT32 Min; // minimum value allowed
+ UINT32 Max; // maximum value allowed
+} NETVADAPTER_ADVANCED_PROPERTY;
+
+#define NETV_OFFSET(field) ((UINT32)FIELD_OFFSET(NetvAdapter,field))
+#define NETV_SIZE(field) RTL_FIELD_SIZE(NetvAdapter,field)
+
+#define CONSTANT_UNICODE_STRING(s) {sizeof( s ) - sizeof( WCHAR ), sizeof( s ), s }
+
+NETVADAPTER_ADVANCED_PROPERTY NetvSupportedProperties[] =
+{
+ // reg value name - Offset in NetvAdapter - Field size - Default Value - Min - Max
+
+ // Standard Keywords
+ { CONSTANT_UNICODE_STRING(L"MACLastByte"), NETV_OFFSET(MACLastByte), NETV_SIZE(MACLastByte), 0, 0, 254 },
+ { CONSTANT_UNICODE_STRING(L"LinkProcIndex"), NETV_OFFSET(LinkProcIndex), NETV_SIZE(LinkProcIndex), 1000, 0, 1023 },
+ { CONSTANT_UNICODE_STRING(L"S0Idle"), NETV_OFFSET(S0Idle), NETV_SIZE(S0Idle), 0, 0, 1 },
+ { CONSTANT_UNICODE_STRING(L"EnableUsoUro"), NETV_OFFSET(EnableUsoUro), NETV_SIZE(EnableUsoUro), 0, 0, 1 },
+};
+
+NTSTATUS
+NetvAdapterReadConfiguration(
+ NetvAdapter *Adapter,
+ WDFDEVICE Device
+ )
+{
+ NTSTATUS status = STATUS_SUCCESS;
+
+ NETCONFIGURATION configuration;
+ RETURN_IF_NOT_STATUS_SUCCESS(
+ NetDeviceOpenConfiguration(Device, WDF_NO_OBJECT_ATTRIBUTES, &configuration));
+
+ // read all the registry values
+ for (auto &property : NetvSupportedProperties)
+ {
+ // Driver should NOT fail the initialization only because it can not
+ // read the registry
+ auto pointer = (PUCHAR)Adapter + property.FieldOffset;
+
+ // Get the configuration value for a specific parameter. Under NT the
+ // parameters are all read in as DWORDs.
+ ULONG value = 0;
+
+ status = NetConfigurationQueryUlong(
+ configuration,
+ NET_CONFIGURATION_QUERY_ULONG_NO_FLAGS,
+ &property.RegName,
+ &value);
+
+ // Store the value in the adapter structure.
+ switch (property.FieldSize)
+ {
+ case 1:
+ *((PUCHAR)pointer) = (UCHAR)value;
+ break;
+
+ case 2:
+ *((PUSHORT)pointer) = (USHORT)value;
+ break;
+
+ case 4:
+ *((PULONG)pointer) = (ULONG)value;
+ break;
+
+ default:
+ break;
+ }
+
+ // If the parameter was present, then check its value for validity.
+ if (NT_SUCCESS(status))
+ {
+ // Check that param value is not too small or too large
+
+ if (value < property.Min ||
+ value > property.Max)
+ {
+ value = property.Default;
+ }
+ }
+ else
+ {
+ value = property.Default;
+ status = STATUS_SUCCESS;
+ }
+ }
+
+ NetConfigurationClose(configuration);
+
+ RETURN_STATUS_SUCCESS();
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/configuration.h b/network/netadaptercx/netvadapterlibrary/code/configuration.h
new file mode 100644
index 00000000..0bfe0170
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/configuration.h
@@ -0,0 +1,8 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+NTSTATUS
+NetvAdapterReadConfiguration(
+ NetvAdapter *Adapter,
+ WDFDEVICE Device
+ );
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/enl.cpp b/network/netadaptercx/netvadapterlibrary/code/enl.cpp
new file mode 100644
index 00000000..709c5a1e
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/enl.cpp
@@ -0,0 +1,1001 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#include "pch.hpp"
+#include <stdlib.h>
+#include "netvadapter.h"
+#include "rxqueue.h"
+#include "txqueue.h"
+#include "trace.h"
+#include "memory.h"
+#include "rtl\KLockHolder.h"
+#include "enl.tmh"
+
+///////////////////////////////////////////////////////////////////////////////
+// BEGIN Generic execution engine thread lib //
+///////////////////////////////////////////////////////////////////////////////
+
+ENL_MLINK NetvEnlMLink[MAX_ADAPTER_COUNT / 2];
+
+/*++
+The iteration routine performs one full pass over all input queues and
+any internal queues (that might be holding previous items waiting to be
+processed).
+--*/
+
+static
+SIZE_T
+CopyTxPacketDataToBuffer(
+ _Out_writes_bytes_(BufferLength) PUCHAR BufferDest,
+ _In_ NET_RING_PACKET_ITERATOR const * Iterator,
+ _In_ NET_EXTENSION const * VirtualAddressExtension,
+ _In_ SIZE_T BufferLength)
+{
+ SIZE_T bytesCopied = 0;
+
+ for (NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(Iterator);
+ NetFragmentIteratorHasAny(&fi) && (BufferLength > 0);
+ NetFragmentIteratorAdvance(&fi))
+ {
+ NET_FRAGMENT const * fragment = NetFragmentIteratorGetFragment(&fi);
+ NET_FRAGMENT_VIRTUAL_ADDRESS const * virtualAddress =
+ NetExtensionGetFragmentVirtualAddress(VirtualAddressExtension, NetFragmentIteratorGetIndex(&fi));
+
+ UCHAR const * pPacketData = (UCHAR const *)virtualAddress->VirtualAddress + fragment->Offset;
+ SIZE_T bytesToCopy =
+ (BufferLength < (SIZE_T) fragment->ValidLength) ? BufferLength : (SIZE_T) fragment->ValidLength;
+ RtlCopyMemory(BufferDest, pPacketData, bytesToCopy);
+
+ bytesCopied += bytesToCopy;
+ BufferDest += bytesToCopy;
+ BufferLength -= bytesToCopy;
+ }
+
+ return bytesCopied;
+}
+
+VOID
+EnlpAffinitizeThread(
+ _In_ ULONG ProcIndex,
+ _In_ ULONG IdealNode
+ )
+{
+ LogInformation(FLAG_DRIVER, L"ProcIndex=%u", ProcIndex);
+ NTSTATUS status;
+ PROCESSOR_NUMBER procNumber = { 0 };
+ GROUP_AFFINITY affinity = { 0 };
+
+ if (ProcIndex == 999)
+ {
+ // No affinity at all
+ return;
+ }
+
+ if (ProcIndex == 1000 || (ProcIndex >= 1001 && ProcIndex <= 1004))
+ {
+ KeQueryNodeActiveAffinity((USHORT)IdealNode, &affinity, NULL);
+
+ if (ProcIndex == 1000)
+ {
+ // Affinitize to all the procs in the Node
+ NOTHING;
+ }
+ else
+ {
+ // Affinitize to the highest numbered proc in the Node for 1001,
+ // next highest numbered proc for 1002, ...
+ ULONG index;
+#ifdef _WIN64
+ NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse64(&index, (ULONG64)affinity.Mask));
+ affinity.Mask = (KAFFINITY)((ULONG64)1 << (index - (ProcIndex - 1001)));
+#else
+ NT_FRE_ASSERTMSG("Failed to find bit", TRUE == BitScanReverse(&index, (ULONG)affinity.Mask));
+ affinity.Mask = (KAFFINITY)((ULONG)1 << (index - (ProcIndex - 1001)));
+#endif
+ }
+ }
+ else
+ {
+ // Affinitize the specifically requested proc
+ status = KeGetProcessorNumberFromIndex(ProcIndex, &procNumber);
+ NT_FRE_ASSERTMSG("Bad ProcIndex", NT_SUCCESS(status));
+
+ affinity.Group = procNumber.Group;
+ affinity.Mask = AFFINITY_MASK(procNumber.Number);
+ }
+
+ EnlThreadSetAffinity(&affinity, NULL);
+}
+
+ENL_START_ROUTINE EnlpThreadRoutine;
+
+NTSTATUS
+EnlpStartThread(
+ _In_ ULONG ProcIndex,
+ _In_ ULONG IdealNode,
+ _In_ ENLP_ITERATION_ROUTINE* IterationRoutine,
+ _In_ PVOID IterationContext,
+ _In_opt_ KAutoEvent* ArmWaitEvent,
+ _Out_ ENLP_THREAD_STATE* EnlThread
+ )
+{
+ LogInformation(FLAG_DRIVER, L"ProcIndex=%u EnlThread=%p", ProcIndex, EnlThread);
+
+ EnlThread->PauseRequested = TRUE;
+ EnlThread->StopRequested = FALSE;
+ EnlThread->IterationRoutine = IterationRoutine;
+ EnlThread->IterationContext = IterationContext;
+ EnlThread->ArmWaitEvent = ArmWaitEvent;
+
+ EnlThread->ProcIndex = ProcIndex;
+ EnlThread->IdealNode = IdealNode;
+
+ RETURN_IF_NOT_STATUS_SUCCESS(
+ EnlThreadCreate(EnlpThreadRoutine, EnlThread, EnlThread->Thread));
+
+ EnlThreadSetPriority(EnlThread->Thread, 15);
+
+ RETURN_STATUS_SUCCESS();
+}
+
+BOOLEAN
+EnlpIsThreadPaused(
+ _In_ CONST ENLP_THREAD_STATE* EnlThread
+ )
+{
+ return EnlThread->PauseRequested;
+}
+
+VOID
+EnlpPauseThread(
+ _Inout_ ENLP_THREAD_STATE* EnlThread
+ )
+{
+ LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread);
+ if (!EnlpIsThreadPaused(EnlThread))
+ {
+ WriteBooleanNoFence(&EnlThread->PauseRequested, TRUE);
+ if (EnlThread->ArmWaitEvent != NULL)
+ {
+ EnlThread->ArmWaitEvent->Set();
+ }
+ EnlThread->PausingEvent.Wait();
+ }
+}
+
+VOID
+EnlpResumeThread(
+ _Inout_ ENLP_THREAD_STATE* EnlThread
+ )
+{
+ LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread);
+ if (EnlpIsThreadPaused(EnlThread))
+ {
+ WriteBooleanNoFence(&EnlThread->PauseRequested, FALSE);
+ KeMemoryBarrier();
+ EnlThread->ResumeEvent.Set();
+ }
+}
+
+VOID
+EnlpStopThread(
+ _Inout_ ENLP_THREAD_STATE* EnlThread
+ )
+{
+ LogInformation(FLAG_DRIVER, L"EnlThread=%p", EnlThread);
+ if (EnlThread->Thread != NULL)
+ {
+ WriteBooleanNoFence(&EnlThread->StopRequested, TRUE);
+ EnlpResumeThread(EnlThread); // in case thread was paused
+ EnlThreadWaitForTermination(EnlThread->Thread);
+ EnlThread->Thread.reset();
+ }
+}
+
+ENL_THREAD_ROUTINE_RETURN
+EnlpThreadRoutine(
+ _In_ PVOID Context
+ )
+{
+ ENLP_THREAD_STATE* enlThread = (ENLP_THREAD_STATE*)Context;
+
+ //
+ // Affinitize the thread
+ //
+ EnlpAffinitizeThread(enlThread->ProcIndex, enlThread->IdealNode);
+
+ //
+ // ENL thread starts at paused state.
+ //
+ enlThread->ResumeEvent.Wait();
+
+ for (;;)
+ {
+
+ if (ReadBooleanNoFence(&enlThread->PauseRequested))
+ {
+ enlThread->PausingEvent.Set();
+ enlThread->ResumeEvent.Wait();
+ }
+
+ if (ReadBooleanNoFence(&enlThread->StopRequested))
+ {
+ break;
+ }
+
+ enlThread->IterationRoutine(enlThread->IterationContext);
+ }
+
+ return ENL_THREAD_ROUTINE_RETURN();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// END Generic execution engine thread lib //
+///////////////////////////////////////////////////////////////////////////////
+
+BOOLEAN
+enlpCheckAndArmQueue(
+ _Inout_ ENLP_QUEUE* Q
+ )
+{
+ BOOLEAN armed = FALSE;
+ auto ringBuffer = Q->Queue->GetPacketRing();
+ // If Q is still empty, take the lock, and if still empty under lock, then
+ // arm it.
+ if (ringBuffer->BeginIndex == ringBuffer->EndIndex)
+ {
+ WdfSpinLockAcquire(Q->Spinlock);
+
+ if (ringBuffer->BeginIndex == ringBuffer->EndIndex)
+ {
+ armed = TRUE;
+ Q->Armed = TRUE;
+ }
+
+ WdfSpinLockRelease(Q->Spinlock);
+ }
+
+ return armed;
+}
+
+VOID
+enlpArmAndWait(
+ _Inout_ ENLP_LINK* EnlLink
+ )
+{
+ ULONG pi, ci;
+
+ for (pi = 0; pi < ENLP_PORT_COUNT; pi++)
+ {
+ ENLP_PORT* port = &EnlLink->Ports[pi];
+
+ for (ci = 0; ci < port->TxQueueCount; ci++)
+ {
+ if (!enlpCheckAndArmQueue(port->TxQueue))
+ {
+ return;
+ }
+ }
+ }
+
+ EnlLink->ArmWaitEvent.Wait();
+}
+
+VOID
+EnlpIterationRoutine(
+ _In_ PVOID IterationContext
+ )
+{
+ auto enlLink = reinterpret_cast<ENLP_LINK *>(IterationContext);
+ bool emptyTx = true;
+ //
+ // Drain up to TX_BATCH_COUNT items from all tx queues first.
+ //
+
+ for (size_t pi = 0U; pi < ENLP_PORT_COUNT; pi++)
+ {
+ auto txport = &enlLink->Ports[pi];
+ auto rxport = &enlLink->Ports[pi ^ 0x1]; // 0 -> 1, 1 -> 0
+
+ for (size_t ci = 0U; ci < txport->TxQueueCount; ci++)
+ {
+ auto txq = &txport->TxQueue[ci];
+ if (txq->State != Started)
+ continue;
+
+ NET_RING_PACKET_ITERATOR txPi = {
+ txq->Queue->m_rings, nullptr, txq->QueueNext, txq->QueueEnd
+ };
+
+ while (NetPacketIteratorHasAny(&txPi))
+ {
+ bool rxDrop = TRUE;
+ auto txPacket = NetPacketIteratorGetPacket(&txPi);
+
+ emptyTx = FALSE;
+
+ if (enlLink->ArmedForWake)
+ {
+ // Save wake packet and trigger a wake signal, any other frames queued after the wake
+ // packet will be dropped
+ enlLink->WakeFrameSize = CopyTxPacketDataToBuffer(
+ &enlLink->WakeFrame[0],
+ &txPi,
+ &txq->TxQueue->VirtualAddressExtension,
+ sizeof(enlLink->WakeFrame));
+#if _KERNEL_MODE
+ enlLink->EvtWakeSignal(enlLink->WakeSignalContext);
+#endif
+
+ // Make sure to disarm wake, otherwise this thread might overwrite the original wake frame
+ EnlDisarmWake(enlLink);
+ }
+
+ if (rxport->RxQueueCount > 0)
+ {
+ //TODO: Currently does 1:1 mapping between Tx and Rx. Need to set up indirection table
+ auto rxq = &rxport->RxQueue[ci];
+
+ if (rxq->State == Started)
+ {
+ NET_RING_FRAGMENT_ITERATOR rxFi = {
+ rxq->Queue->m_rings, nullptr, rxq->QueueNext, rxq->QueueEnd
+ };
+
+ auto rxPi = NetRingGetPostPackets(rxq->Queue->m_rings);
+
+ if (NetFragmentIteratorHasAny(&rxFi) && NetPacketIteratorHasAny(&rxPi))
+ {
+ rxDrop = FALSE;
+
+ auto fragment = NetFragmentIteratorGetFragment(&rxFi);
+ BYTE* fragmentBuffer = nullptr;
+
+ auto const rxVirtualAddress =
+ NetExtensionGetFragmentVirtualAddress(
+ &rxq->RxQueue->VirtualAddressExtension,
+ NetFragmentIteratorGetIndex(&rxFi));
+
+ fragmentBuffer = static_cast<BYTE *>(rxVirtualAddress->VirtualAddress);
+
+ fragment->Offset = 0;
+ fragment->ValidLength =
+ CopyTxPacketDataToBuffer(
+ fragmentBuffer,
+ &txPi,
+ &txq->TxQueue->VirtualAddressExtension,
+ static_cast<SIZE_T>(fragment->Capacity));
+
+ auto rxPacket = NetPacketIteratorGetPacket(&rxPi);
+ rxPacket->FragmentIndex = NetFragmentIteratorGetIndex(&rxFi);
+ rxPacket->FragmentCount = 1;
+
+ if (enlLink->InitializePacketLayout)
+ {
+ rxPacket->Layout = txPacket->Layout;
+ }
+ else
+ {
+ rxPacket->Layout = {};
+ }
+
+ if (rxq->RxQueue->RxXSumExtension.Enabled)
+ {
+ auto RxXSum = NetExtensionGetPacketChecksum(
+ &rxq->RxQueue->RxXSumExtension,
+ NetPacketIteratorGetIndex(&rxPi));
+ RxXSum->Layer2 = NetPacketRxChecksumEvaluationNotChecked;
+ RxXSum->Layer3 = NetPacketRxChecksumEvaluationValid;
+ RxXSum->Layer4 = NetPacketRxChecksumEvaluationValid;
+ }
+
+ if (rxq->RxQueue->UdpRscExtension.Enabled &&
+ txq->TxQueue->UsoExtension.Enabled)
+ {
+ auto txUso = NetExtensionGetPacketGso(
+ &txq->TxQueue->UsoExtension,
+ NetPacketIteratorGetIndex(&txPi));
+ if (txPacket->Layout.Layer4Type == NetPacketLayer4TypeUdp &&
+ txUso->UDP.Mss > 0)
+ {
+ if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv6NoExtensions)
+ {
+ UINT16* ipv6PayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 4);
+ *ipv6PayloadLength = _byteswap_ushort(
+ (USHORT)(fragment->ValidLength -
+ (txPacket->Layout.Layer2HeaderLength + txPacket->Layout.Layer3HeaderLength)));
+ }
+ else if (txPacket->Layout.Layer3Type == NetPacketLayer3TypeIPv4NoOptions)
+ {
+ UINT16* ipv4TotalLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength + 2);
+ *ipv4TotalLength = _byteswap_ushort((USHORT)(fragment->ValidLength - txPacket->Layout.Layer2HeaderLength ));
+ NT_FRE_ASSERT(*ipv4TotalLength > 0);
+ }
+
+ UINT16* udpPayloadLength = (UINT16*)(fragmentBuffer + txPacket->Layout.Layer2HeaderLength +
+ txPacket->Layout.Layer3HeaderLength + 4);
+ *udpPayloadLength = _byteswap_ushort(
+ (USHORT)(fragment->ValidLength -
+ (txPacket->Layout.Layer3HeaderLength + txPacket->Layout.Layer2HeaderLength)));
+
+ // Set the checksum to zero
+ UINT16* udpChecksum = udpPayloadLength + 1;
+ *udpChecksum = 0;
+
+ auto rxUro = NetExtensionGetPacketRsc(
+ &rxq->RxQueue->UdpRscExtension,
+ NetPacketIteratorGetIndex(&rxPi));
+ rxUro->UDP.CoalescedSegmentSize = (UINT16)txUso->UDP.Mss;
+ rxUro->UDP.CoalescedSegmentCount =
+ (UINT16)((fragment->ValidLength - txPacket->Layout.Layer3HeaderLength - txPacket->Layout.Layer4HeaderLength
+ + txUso->UDP.Mss - 1) / txUso->UDP.Mss);
+ NT_FRE_ASSERT(rxUro->UDP.CoalescedSegmentCount > 1);
+ NT_FRE_ASSERT(*udpPayloadLength >= 8);
+ }
+ }
+
+ // prevent any reordering the tx/rx completion flag
+ KeMemoryBarrier();
+
+ // Use Scratch field as completion flag for the rx fragment
+ fragment->Scratch = 1;
+ rxPacket->Scratch = 1;
+
+ NetFragmentIteratorAdvance(&rxFi);
+ NetPacketIteratorAdvance(&rxPi);
+
+ rxq->QueueNext = NetFragmentIteratorGetIndex(&rxFi);
+ NetPacketIteratorSet(&rxPi);
+
+ if (rxq->Notify)
+ {
+ rxport->Interrupt(rxq->Queue->m_handle, rxq->TxRx);
+ }
+ }
+ }
+ }
+
+ if (rxDrop)
+ {
+ // TODO - add rxdrop stat
+ }
+
+ // Use Scratch field as completion flag for the tx packet
+ txPacket->Scratch = 1;
+ NetPacketIteratorAdvance(&txPi);
+ }
+
+ if (!emptyTx)
+ {
+ if (txq->Notify)
+ {
+ txport->Interrupt(txq->Queue->m_handle, txq->TxRx);
+ }
+
+ // Store the next index so that the EnlThread knows which packet to start from in the next iteration
+ txq->QueueNext = NetPacketIteratorGetIndex(&txPi);
+ }
+ }
+ }
+
+ ULONG64 ts;
+
+ if (emptyTx)
+ {
+ if (enlLink->Poll == FALSE)
+ {
+ enlpArmAndWait(enlLink);
+ }
+
+ ts = ReadTimeStampCounter();
+ enlLink->EmptyTicks += (ts - enlLink->Ts);
+ }
+ else
+ {
+ ts = ReadTimeStampCounter();
+ enlLink->BusyTicks += (ts - enlLink->Ts);
+ }
+
+ enlLink->Ts = ts;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+ENLP_QUEUE *
+EnlCreateQueue(
+ _In_ NETPACKETQUEUE Queue,
+ _In_ BOOLEAN Tx
+ )
+{
+ ENLP_LINK* enlLink;
+ ENLP_PORT* port;
+ ENLP_QUEUE* enlQueue;
+ NTSTATUS status;
+
+ if (Tx) // Tx
+ {
+ NetvTxQueue* netvTxQueue = NetvTxQueueGetContext(Queue);
+ enlLink = NetvEnlMLink[netvTxQueue->m_adapter.EnlIndex].LinkHandle[0];
+ port = &enlLink->Ports[netvTxQueue->m_adapter.EnlPortIndex];
+ enlQueue = &port->TxQueue[0]; // Change to accommodate multiple Queues
+ enlQueue->Queue = netvTxQueue;
+ enlQueue->TxQueue = netvTxQueue;
+ enlQueue->State = Stopped;
+ enlQueue->ArmWaitEvent = &enlLink->ArmWaitEvent;
+ enlQueue->QueueNext = 0;
+ enlQueue->QueueEnd = 0;
+ enlQueue->TxRx = TX;
+ enlQueue->EnlPortHandle = port;
+ port->TxQueueCount++;
+ LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu TxQueue=%p",
+ &netvTxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvTxQueue);
+ }
+ else // Rx
+ {
+ NetvRxQueue* netvRxQueue = NetvRxQueueGetContext(Queue);
+ enlLink = NetvEnlMLink[netvRxQueue->m_adapter.EnlIndex].LinkHandle[0];
+ port = &enlLink->Ports[netvRxQueue->m_adapter.EnlPortIndex];
+ enlQueue = &port->RxQueue[0]; // Change to accommodate multiple Queues
+ enlQueue->Queue = netvRxQueue;
+ enlQueue->RxQueue = netvRxQueue;
+ enlQueue->State = Stopped;
+ enlQueue->QueueNext = 0;
+ enlQueue->QueueEnd = 0;
+ enlQueue->TxRx = RX;
+ enlQueue->EnlPortHandle = port;
+ port->RxQueueCount++;
+ LogInformation(FLAG_DRIVER, L"Adapter=%p Queue=%Iu RxQueue=%p",
+ &netvRxQueue->m_adapter, reinterpret_cast<ULONG_PTR>(Queue), netvRxQueue);
+ }
+
+ // Create WDF spinlock for the queue, parent to the queue's WDF handle
+ WDF_OBJECT_ATTRIBUTES attributes;
+ WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
+ attributes.ParentObject = enlQueue->Queue->m_handle;
+ status = WdfSpinLockCreate(&attributes, &enlQueue->Spinlock);
+ NT_ASSERT(NT_SUCCESS(status));
+
+
+ EnlpResumeThread(&enlLink->EnlThread);
+
+ return enlQueue;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlDestroyQueue(
+ _In_ ENLP_QUEUE * Queue,
+ _In_ BOOLEAN Tx
+ )
+{
+ LogInformation(FLAG_DRIVER, L"Queue=%p", Queue);
+ ENLP_PORT* port = Queue->EnlPortHandle;
+
+ if (Tx)
+ {
+ port->TxQueueCount--;
+ }
+ else
+ {
+ port->RxQueueCount--;
+ }
+
+ Queue->ArmWaitEvent = nullptr;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlRingDoorBell(
+ _In_ ENLP_QUEUE * Queue,
+ _In_ ULONG EndIndex
+ )
+{
+ InterlockedExchange((volatile LONG *)&Queue->QueueEnd, (LONG)EndIndex);
+
+ WdfSpinLockAcquire(Queue->Spinlock);
+
+ if (Queue->Armed)
+ {
+ NT_ASSERT(Queue->ArmWaitEvent != NULL);
+ Queue->Armed = FALSE;
+ WdfSpinLockRelease(Queue->Spinlock);
+ Queue->ArmWaitEvent->Set();
+ return;
+ }
+
+ WdfSpinLockRelease(Queue->Spinlock);
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlArmInterrupt(
+ _In_ ENLP_QUEUE * Queue,
+ _In_ BOOLEAN notificationEnabled
+ )
+{
+ Queue->Notify = notificationEnabled;
+
+ // TODO: InterlockedExchange reduces the throughput of the ENL from about 500MB/s to 8MB/s. -> why
+ //InterlockedExchange((volatile LONG *)&Queue->Notify, (LONG)notificationEnabled);
+}
+
+_Use_decl_annotations_
+VOID
+EnlIndicateQueueState(
+ ENLP_QUEUE * Queue,
+ ENL_QUEUE_STATE State
+ )
+{
+ Queue->State = State;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlCreateLink(
+ _In_ ULONG ProcessorIndex,
+ _In_ ULONG IdealNode,
+ _In_ BOOLEAN Poll,
+ _In_ BOOLEAN InitializePacketLayout,
+ _Out_ ENLP_LINK ** EnlLink
+ )
+{
+ LogInformation(FLAG_DRIVER, L"ProcessorIndex=%u", ProcessorIndex);
+
+ auto enlLink = wil::make_unique_nothrow<ENLP_LINK>();
+ RETURN_NTSTATUS_IF(
+ STATUS_INSUFFICIENT_RESOURCES,
+ ! enlLink);
+
+ enlLink->Ts = ReadTimeStampCounter();
+ enlLink->Poll = Poll;
+ enlLink->InitializePacketLayout = InitializePacketLayout;
+
+ RETURN_IF_NOT_STATUS_SUCCESS(
+ EnlpStartThread(
+ ProcessorIndex,
+ IdealNode,
+ EnlpIterationRoutine,
+ enlLink.get(),
+ enlLink->Poll ? NULL : &enlLink->ArmWaitEvent,
+ &enlLink->EnlThread));
+
+ *EnlLink = enlLink.release();
+
+ RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlIsPortActive(
+ _In_ ENLP_LINK * EnlLink,
+ _In_ ULONG PortIndex
+ )
+{
+ ENLP_PORT* port = &EnlLink->Ports[PortIndex];
+ ENLP_QUEUE* txq = &port->TxQueue[0];
+ NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT);
+
+ return (txq->Queue == nullptr) ? FALSE : TRUE;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlIsLinkActive(
+ _In_ ENLP_LINK * EnlLink
+ )
+{
+ return (EnlIsPortActive(EnlLink, 0) || EnlIsPortActive(EnlLink, 1));
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlActivateLinkPort(
+ _In_ ENLP_LINK * EnlLink,
+ _In_ ULONG PortIndex,
+ _In_ ENL_INTERRUPT_ROUTINE Interrupt,
+ _In_ PVOID PortContext
+ )
+{
+ ENLP_PORT* port = &EnlLink->Ports[PortIndex];
+
+ LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue);
+
+ NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, PortIndex));
+ NT_FRE_ASSERT(port->TxQueueCount == 0);
+ NT_FRE_ASSERT(port->RxQueueCount == 0);
+
+ port->Interrupt = Interrupt;
+ port->PortContext = PortContext;
+
+ RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlDeactivateLinkPort(
+ _In_ ENLP_LINK * EnlLink,
+ _In_ ULONG PortIndex
+ )
+{
+ NT_FRE_ASSERT(PortIndex < ENLP_PORT_COUNT);
+
+ ENLP_PORT* port = &EnlLink->Ports[PortIndex];
+ LogInformation(FLAG_DRIVER, L"PortIndex=%u Queue=%p", PortIndex, port->TxQueue[0].Queue);
+
+ NT_FRE_ASSERT(EnlIsPortActive(EnlLink, PortIndex));
+
+ KLockThisExclusive(EnlLink->Lock);
+ NT_FRE_ASSERT(!EnlpIsThreadPaused(&EnlLink->EnlThread));
+ EnlpPauseThread(&EnlLink->EnlThread);
+
+#if _KERNEL_MODE
+ KeFlushQueuedDpcs();
+#endif
+
+ port->PortContext = NULL;
+
+ //Clears reference to only first queue - Change for all queues
+ port->TxQueue[0].Queue = nullptr;
+ port->RxQueue[0].Queue = nullptr;
+
+
+ // As long as there's one port with active queues, the EnlThread will run.
+ for (ULONG i = 0; i < ENLP_PORT_COUNT; i++)
+ {
+ if (EnlIsPortActive(EnlLink, i))
+ {
+ EnlpResumeThread(&EnlLink->EnlThread);
+ break;
+ }
+ }
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlCloseLink(
+ _In_ ENLP_LINK * EnlLink
+ )
+{
+ ULONG i;
+
+ for (i = 0; i < ENLP_PORT_COUNT; i++)
+ {
+ NT_FRE_ASSERT(!EnlIsPortActive(EnlLink, i));
+ }
+
+ EnlpStopThread(&EnlLink->EnlThread);
+
+ delete EnlLink;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Multi link wrapper APIs //
+///////////////////////////////////////////////////////////////////////////////
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlMCreateLink(
+ _In_range_(1, ENL_MLINK_MAX)ULONG LinkCount,
+ _In_reads_(LinkCount) ULONG ProcessorIndex,
+ _In_ BOOLEAN Poll,
+ _Out_ ENL_MLINK* EnlMLink
+ )
+{
+ NTSTATUS status = STATUS_SUCCESS;
+ ULONG i = 0;
+
+ RtlZeroMemory(EnlMLink, sizeof(*EnlMLink));
+
+ if (LinkCount < 1 || LinkCount > ENL_MLINK_MAX ||
+ (LinkCount & (LinkCount - 1)) != 0)
+ {
+ status = STATUS_REQUEST_NOT_ACCEPTED;
+ goto exit;
+ }
+
+ for (i = 0; i < LinkCount; i++)
+ {
+#ifdef _KERNEL_MODE
+ const BOOLEAN initializePacketLayout = TRUE;
+#else
+ const BOOLEAN initializePacketLayout = FALSE;
+#endif
+ status = EnlCreateLink(ProcessorIndex, 0, Poll, initializePacketLayout, &EnlMLink->LinkHandle[i]);
+
+ if (!NT_SUCCESS(status))
+ {
+ goto exit;
+ }
+ }
+
+ EnlMLink->LinkCount = LinkCount;
+
+exit:
+
+ if (!NT_SUCCESS(status))
+ {
+ for (; i > 0; i--)
+ {
+ EnlCloseLink(EnlMLink->LinkHandle[i - 1]);
+ EnlMLink->LinkHandle[i - 1] = NULL;
+ }
+ }
+ RETURN_NTSTATUS_IF(
+ status,
+ status != STATUS_SUCCESS);
+
+ RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void EnlSetPdoWakeSignalCallback(
+ _In_ ENLP_LINK * EnlLink,
+ _In_ EVT_ENLP_PDO_WAKE_SIGNAL* evtPdoWakeSignal,
+ _In_ void* Context)
+{
+ EnlLink->EvtWakeSignal = evtPdoWakeSignal;
+ EnlLink->WakeSignalContext = Context;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlArmWake(
+ _In_ ENLP_LINK * EnlLink
+)
+{
+ EnlLink->ArmedForWake = TRUE;
+ RtlZeroMemory(&EnlLink->WakeFrame[0], sizeof(EnlLink->WakeFrame));
+ EnlLink->WakeFrameSize = 0;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlDisarmWake(
+ _In_ ENLP_LINK * EnlLink
+)
+{
+ EnlLink->ArmedForWake = FALSE;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+size_t
+EnlCopyWakeFrame(
+ _In_ ENLP_LINK * EnlLink,
+ _Out_writes_bytes_(BufferSize) unsigned char * Buffer,
+ _In_ size_t BufferSize
+)
+{
+ if (EnlLink->WakeFrameSize == 0)
+ {
+ // There was no wake
+ return 0;
+ }
+
+ if (BufferSize < EnlLink->WakeFrameSize)
+ {
+ // Wake frame is larger than what we can indicate
+ return 0;
+ }
+
+ RtlCopyMemory(Buffer, &EnlLink->WakeFrame[0], EnlLink->WakeFrameSize);
+ auto const wakeFrameSize = EnlLink->WakeFrameSize;
+
+ // Make sure to erase the wake frame, since the network interface might have
+ // multiple receive queues
+ RtlZeroMemory(&EnlLink->WakeFrame[0], EnlLink->WakeFrameSize);
+ EnlLink->WakeFrameSize = 0;
+
+ return wakeFrameSize;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlMIsPortActive(
+ _In_ CONST ENL_MLINK* EnlMLink,
+ _In_ ULONG PortIndex
+ )
+{
+ ULONG i;
+
+ BOOLEAN result = EnlIsPortActive(EnlMLink->LinkHandle[0], PortIndex);
+
+ for (i = 1; i < EnlMLink->LinkCount; i++)
+ {
+ NT_FRE_ASSERT(EnlIsPortActive(EnlMLink->LinkHandle[i], PortIndex) == result);
+ }
+
+ return result;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlMIsLinkActive(
+ _In_ CONST ENL_MLINK* EnlMLink
+ )
+{
+ ULONG i;
+
+ BOOLEAN result = EnlIsLinkActive(EnlMLink->LinkHandle[0]);
+
+ for (i = 1; i < EnlMLink->LinkCount; i++)
+ {
+ NT_FRE_ASSERT(EnlIsLinkActive(EnlMLink->LinkHandle[i]) == result);
+ }
+
+ return result;
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlMActivateLinkPort(
+ _In_ CONST ENL_MLINK* EnlMLink,
+ _In_ ULONG PortIndex,
+ _In_ ENL_INTERRUPT_ROUTINE Interrupt,
+ _In_ PVOID PortContext
+ )
+{
+ LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex);
+
+ NTSTATUS status = STATUS_SUCCESS;
+ ULONG i;
+
+ for (i = 0; i < EnlMLink->LinkCount; i++)
+ {
+ status = EnlActivateLinkPort(
+ EnlMLink->LinkHandle[i],
+ PortIndex,
+ Interrupt,
+ PortContext);
+
+ if (!NT_SUCCESS(status))
+ {
+ break;
+ }
+ }
+
+ if (!NT_SUCCESS(status))
+ {
+ for (; i > 0; i--)
+ {
+ EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex);
+ }
+ }
+
+ RETURN_NTSTATUS_IF(status,
+ status != STATUS_SUCCESS);
+
+ RETURN_STATUS_SUCCESS();
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlMDeactivateLinkPort(
+ _In_ CONST ENL_MLINK* EnlMLink,
+ _In_ ULONG PortIndex
+ )
+{
+ LogInformation(FLAG_DRIVER, L"EnlMLink=%p PortIndex=%u", EnlMLink, PortIndex);
+
+ ULONG i;
+
+ for (i = 0; i < EnlMLink->LinkCount; i++)
+ {
+ EnlDeactivateLinkPort(EnlMLink->LinkHandle[i], PortIndex);
+ }
+}
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlMCloseLink(
+ _Inout_ ENL_MLINK* EnlMLink
+ )
+{
+ LogInformation(FLAG_DRIVER, L"EnlMLink=%p", EnlMLink);
+
+ ULONG i;
+
+ for (i = 0; i < EnlMLink->LinkCount; i++)
+ {
+ EnlCloseLink(EnlMLink->LinkHandle[i]);
+ EnlMLink->LinkHandle[i] = NULL;
+ }
+
+ EnlMLink->LinkCount = 0;
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/enl.h b/network/netadaptercx/netvadapterlibrary/code/enl.h
new file mode 100644
index 00000000..2927a582
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/enl.h
@@ -0,0 +1,264 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+//
+// Emulated Network Link (ENL) definitions
+//
+
+//
+// An ENL connects two virtual network adapters directly. Packets sent over
+// one adapter are delivered to the other adapter and vice versa.
+//
+// An ENL is created with a processor index. This processor is used to emulate
+// the NIC hardware for the adapters connected by the ENL.
+//
+// Caller can send packets as NBLs over a given port, and also receive
+// incoming packets placed into NBLs over a given port.
+//
+// ENL indicates tx and rx NBL completions over the target processor(s)
+// determined by the RSS indirection table and the hash value for each NBL.
+// The current version of the ENL requires a symmetric Toeplitz hash key
+// to be used by the system globally so that both directions of a given
+// 4-tuple (or 2-tuple) yield the same hash value. ENL currently indicates
+// NBL completions by queueing DPCs to the target processor.
+//
+
+
+#ifndef _KERNEL_MODE
+#define ASSERT(x) NT_ASSERT(x)
+
+#endif
+#include "rtl/KWaitEvent.h"
+#include "rtl/KPushLock.h"
+
+#define ENL_MAX_PROC_COUNT 16
+#define ENLP_PORT_COUNT 2
+
+#define TX TRUE
+#define RX FALSE
+
+enum ENL_QUEUE_STATE {
+ Stopped,
+ Started
+ };
+
+typedef
+VOID
+(ENL_INTERRUPT_ROUTINE) (
+ _Inout_ PVOID PortContext,
+ _In_ bool TxRx
+ );
+
+struct ENLP_LINK;
+struct DECLSPEC_ALIGN(PAGE_SIZE) ENLP_PORT;
+struct ENLP_QUEUE;
+
+typedef NTSTATUS(EVT_ENLP_PDO_WAKE_SIGNAL)(_In_ void* Context);
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+ENLP_QUEUE *
+EnlCreateQueue(
+ _In_ NETPACKETQUEUE Queue,
+ _In_ BOOLEAN Tx
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlDestroyQueue(
+ _In_ ENLP_QUEUE * QueueContext,
+ _In_ BOOLEAN Tx
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlRingDoorBell(
+ _In_ ENLP_QUEUE * QueueContext,
+ _In_ ULONG EndIndex
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlArmInterrupt(
+ _In_ ENLP_QUEUE * QueueContext,
+ _In_ BOOLEAN NotificationEnabled
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlSetPdoWakeSignalCallback(
+ _In_ ENLP_LINK * EnlLinkHandle,
+ _In_ EVT_ENLP_PDO_WAKE_SIGNAL* evtPdoWakeSignal,
+ _In_ void* Context
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlArmWake(
+ _In_ ENLP_LINK * EnlLinkHandle
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlDisarmWake(
+ _In_ ENLP_LINK * EnlLinkHandle
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+size_t
+EnlCopyWakeFrame(
+ _In_ ENLP_LINK * EnlLinkHandle,
+ _Out_writes_bytes_(BufferSize) unsigned char * Buffer,
+ _In_ size_t BufferSize
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlIsLinkActive(
+ _In_ ENLP_LINK * EnlLinkHandle
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlIndicateQueueState(
+ _In_ ENLP_QUEUE * EnlLinkHandle,
+ _In_ ENL_QUEUE_STATE State
+ );
+
+///////////////////////////////////////////////////////////////////////////////
+// Multi link wrapper APIs //
+///////////////////////////////////////////////////////////////////////////////
+
+#define ENL_MLINK_MAX 4
+
+typedef struct
+{
+ _Field_range_(1, ENL_MLINK_MAX) ULONG LinkCount;
+ _Field_size_(LinkCount) ENLP_LINK * LinkHandle[ENL_MLINK_MAX];
+} ENL_MLINK;
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlMCreateLink(
+ _In_range_(1, ENL_MLINK_MAX)ULONG LinkCount,
+ _In_reads_(LinkCount) ULONG ProcessorIndex,
+ _In_ BOOLEAN Poll,
+ _Out_ ENL_MLINK* EnlMLink
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+EnlMIsPortActive(
+ _In_ CONST ENL_MLINK* EnlMLink,
+ _In_ ULONG PortIndex
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+NTSTATUS
+EnlMActivateLinkPort(
+ _In_ CONST ENL_MLINK* EnlMLink,
+ _In_ ULONG PortIndex,
+ _In_ ENL_INTERRUPT_ROUTINE Interrupt,
+ _In_ PVOID PortContext
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+EnlMDeactivateLinkPort(
+ _In_ CONST ENL_MLINK* EnlMLink,
+ _In_ ULONG PortIndex
+ );
+
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+EnlMCloseLink(
+ _Inout_ ENL_MLINK* EnlMLink
+ );
+
+#define ENL_MAXIMUM_WAKE_FRAME_SIZE 1514
+
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+VOID
+(ENLP_ITERATION_ROUTINE) (
+ _In_ PVOID IterationContext
+ );
+
+struct ENLP_THREAD_STATE
+{
+ BOOLEAN PauseRequested{};
+ BOOLEAN StopRequested{};
+ ENLP_ITERATION_ROUTINE* IterationRoutine{};
+ PVOID IterationContext{};
+ KAutoEvent *ArmWaitEvent{};
+ KAutoEvent ResumeEvent{};
+ KAutoEvent PausingEvent{};
+ ULONG ProcIndex{};
+ ULONG IdealNode{};
+ unique_thread Thread{};
+};
+
+class NetvQueue;
+class NetvRxQueue;
+class NetvTxQueue;
+
+struct ENLP_QUEUE
+{
+ BOOLEAN TxRx{};
+ BOOLEAN Notify{};
+ ULONG QueueEnd{};
+ ULONG QueueNext{};
+
+ ENLP_PORT * EnlPortHandle{};
+
+ BOOLEAN Armed{};
+ WDFSPINLOCK Spinlock{};
+ KAutoEvent *ArmWaitEvent{};
+
+ ENL_QUEUE_STATE State{};
+
+ NetvQueue * Queue{nullptr};
+ union {
+ NetvRxQueue * RxQueue;
+ NetvTxQueue * TxQueue;
+ };
+
+};
+
+struct DECLSPEC_ALIGN(PAGE_SIZE) ENLP_PORT
+{
+ ENL_INTERRUPT_ROUTINE* Interrupt;
+ PVOID PortContext;
+ ULONG TxQueueCount;
+ ULONG RxQueueCount;
+
+ DECLSPEC_CACHEALIGN
+ ENLP_QUEUE TxQueue[ENL_MAX_PROC_COUNT];
+
+ DECLSPEC_CACHEALIGN
+ ENLP_QUEUE RxQueue[ENL_MAX_PROC_COUNT];
+};
+
+struct ENLP_LINK
+{
+ KPushLock Lock{};
+ ULONG64 Ts{};
+ ULONG64 BusyTicks{};
+ ULONG64 EmptyTicks{};
+ BOOLEAN Poll{};
+ BOOLEAN InitializePacketLayout{};
+
+ //
+ // Power related features. If PowerInterface is NULL none of the other
+ // fields have meaning
+ //
+ EVT_ENLP_PDO_WAKE_SIGNAL* EvtWakeSignal;
+ void* WakeSignalContext;
+ BOOLEAN ArmedForWake{};
+ unsigned char WakeFrame[ENL_MAXIMUM_WAKE_FRAME_SIZE];
+ size_t WakeFrameSize{};
+
+ KAutoEvent ArmWaitEvent{};
+ ENLP_THREAD_STATE EnlThread{};
+ ENLP_PORT Ports[ENLP_PORT_COUNT];
+};
+#define MAX_ADAPTER_COUNT 2
+extern ENL_MLINK NetvEnlMLink[MAX_ADAPTER_COUNT / 2];
+C_ASSERT((ENLP_PORT_COUNT & 0x1) == 0);
diff --git a/network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp b/network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp
new file mode 100644
index 00000000..ffe70d6e
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/enlthreads.cpp
@@ -0,0 +1,136 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#include "pch.hpp"
+//#include <ntassert.h>
+#include "enlthreads.h"
+
+#ifdef _KERNEL_MODE
+unique_thread::operator bool(
+ void
+) const
+{
+ return !!NtHandle;
+}
+
+void unique_thread::reset(
+)
+{
+ NtHandle.reset();
+ ObHandle.reset();
+}
+#endif
+
+ENL_THREAD
+EnlGetCurrentThread(
+ void
+)
+{
+#ifdef _KERNEL_MODE
+ return KeGetCurrentThread();
+#else
+ return GetCurrentThreadId();
+#endif
+}
+
+_Use_decl_annotations_
+NTSTATUS
+EnlThreadCreate(
+ ENL_START_ROUTINE StartRoutine,
+ void * Context,
+ unique_thread & Thread
+)
+{
+#ifdef _KERNEL_MODE
+
+ unique_zw_handle ntHandle;
+ unique_pkthread obHandle;
+
+ auto const ntStatus = PsCreateSystemThread(
+ &ntHandle,
+ THREAD_ALL_ACCESS,
+ nullptr,
+ nullptr,
+ nullptr,
+ StartRoutine,
+ Context);
+
+ if (ntStatus != STATUS_SUCCESS)
+ {
+ return ntStatus;
+ }
+
+ NT_FRE_ASSERT(
+ NT_SUCCESS(
+ ObReferenceObjectByHandle(
+ ntHandle.get(),
+ THREAD_ALL_ACCESS,
+ nullptr,
+ KernelMode,
+ reinterpret_cast<void **>(&obHandle),
+ nullptr)));
+
+ Thread.NtHandle = wistd::move(ntHandle);
+ Thread.ObHandle = wistd::move(obHandle);
+
+#else
+ wil::unique_handle thread{ CreateThread(nullptr, 0, StartRoutine, Context, 0, nullptr) };
+
+ if (!thread)
+ {
+ return NTSTATUS_FROM_WIN32(GetLastError());
+ }
+
+ Thread = wistd::move(thread);
+#endif
+
+ return STATUS_SUCCESS;
+}
+
+void
+EnlThreadSetPriority(
+ unique_thread & Thread,
+ ENL_THREAD_PRIORITY Priority
+)
+{
+#ifdef _KERNEL_MODE
+ // KeSetBasePriorityThread does not take the actual priority, but an increment
+ // to be added to the current base priority. Calculate this value.
+ auto const increment = Priority - (LOW_REALTIME_PRIORITY + LOW_PRIORITY) / 2;
+ KeSetBasePriorityThread(Thread.ObHandle.get(), increment);
+#else
+ SetThreadPriority(Thread.get(), Priority);
+#endif
+}
+
+_Use_decl_annotations_
+void
+EnlThreadSetAffinity(
+ PGROUP_AFFINITY GroupAffinity,
+ PGROUP_AFFINITY PreviousAffinity
+)
+{
+#ifdef _KERNEL_MODE
+ KeSetSystemGroupAffinityThread(GroupAffinity, PreviousAffinity);
+#else
+ SetThreadGroupAffinity(GetCurrentThread(), GroupAffinity, PreviousAffinity);
+#endif
+}
+
+_Use_decl_annotations_
+void
+EnlThreadWaitForTermination(
+ unique_thread & Thread
+)
+{
+#ifdef _KERNEL_MODE
+ KeWaitForSingleObject(
+ Thread.ObHandle.get(),
+ KWAIT_REASON::Executive,
+ KernelMode,
+ FALSE,
+ nullptr);
+#else
+ WaitForSingleObject(
+ Thread.get(),
+ INFINITE);
+#endif
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/enlthreads.h b/network/netadaptercx/netvadapterlibrary/code/enlthreads.h
new file mode 100644
index 00000000..062d728e
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/enlthreads.h
@@ -0,0 +1,119 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#pragma once
+
+#include <wil/resource.h>
+// #include <KMacros.h>
+
+#ifdef _KERNEL_MODE
+ using ENL_START_ROUTINE = KSTART_ROUTINE;
+ using ENL_THREAD_ROUTINE_RETURN = VOID;
+ using ENL_THREAD = PKTHREAD;
+ // static const EC_THREAD EC_THREAD_INVALID = nullptr;
+ using ENL_THREAD_PRIORITY = LONG;
+ // #define EC_INVALID_THREAD_PRIORITY MAXLONG
+
+ using unique_zw_handle = wil::unique_any<HANDLE, decltype(&::ZwClose), &::ZwClose>;
+ using unique_pkthread = wil::unique_any<PKTHREAD, decltype(&ObfDereferenceObject), &ObfDereferenceObject>;
+ using unique_pkevent = wil::unique_any<PKEVENT, decltype(&ObfDereferenceObject), &ObfDereferenceObject>;
+ using unique_completionport = wil::unique_any<void *, decltype(&ObfDereferenceObject), &ObfDereferenceObject>;
+
+ struct unique_thread
+ {
+ unique_zw_handle
+ NtHandle;
+
+ unique_pkthread
+ ObHandle;
+
+ operator bool(
+ void
+ ) const;
+
+ void reset(
+ );
+ };
+#else
+
+#ifndef NOTHING
+#define NOTHING
+#endif
+
+#ifndef AFFINITY_MASK
+#define AFFINITY_MASK(n) ((KAFFINITY)1 << (n))
+#endif
+
+typedef struct _NDTBUS_POWER_INTERFACE_STANDARD
+{
+} NDTBUS_POWER_INTERFACE_STANDARD;
+
+_IRQL_requires_same_
+typedef DWORD (WINAPI ENL_START_ROUTINE)(
+ LPVOID lpThreadParameter
+);
+using ENL_THREAD_ROUTINE_RETURN = DWORD;
+using ENL_THREAD = DWORD;
+// static const EC_THREAD EC_THREAD_INVALID = 0;
+using ENL_THREAD_PRIORITY = int;
+// #define EC_INVALID_THREAD_PRIORITY THREAD_PRIORITY_ERROR_RETURN
+using unique_thread = wil::unique_handle;
+
+#define KeGetProcessorIndexFromNumber(_processor) (_processor)->Number
+#define KeMemoryBarrier() MemoryBarrier()
+
+inline
+NTSTATUS
+KeGetProcessorNumberFromIndex (
+ _In_ ULONG ProcIndex,
+ _Out_ PPROCESSOR_NUMBER ProcNumber
+ )
+{
+ ProcNumber->Number = static_cast<UCHAR>(ProcIndex);
+ ProcNumber->Group = 0;
+ ProcNumber->Reserved = 0;
+
+ return STATUS_SUCCESS;
+}
+
+inline
+VOID
+KeQueryNodeActiveAffinity (
+ __in USHORT NodeNumber,
+ __out_opt PGROUP_AFFINITY Affinity,
+ __out_opt PUSHORT Count
+)
+{
+ UNREFERENCED_PARAMETER(NodeNumber);
+ UNREFERENCED_PARAMETER(Count);
+
+ GetThreadGroupAffinity(GetCurrentThread(), Affinity);
+}
+#endif
+
+ENL_THREAD
+EnlGetCurrentThread(
+ void
+);
+
+NTSTATUS
+EnlThreadCreate(
+ _In_ ENL_START_ROUTINE StartRoutine,
+ _In_opt_ void * Context,
+ _Out_ unique_thread & Thread
+);
+
+void
+EnlThreadSetPriority(
+ _In_ unique_thread & Thread,
+ _In_ ENL_THREAD_PRIORITY Priority
+);
+
+void
+EnlThreadSetAffinity(
+ _In_ PGROUP_AFFINITY GroupAffinity,
+ _Out_opt_ PGROUP_AFFINITY PreviousAffinity
+);
+
+void
+EnlThreadWaitForTermination(
+ _In_ unique_thread & Thread
+);
diff --git a/network/netadaptercx/netvadapterlibrary/code/memory.cpp b/network/netadaptercx/netvadapterlibrary/code/memory.cpp
new file mode 100644
index 00000000..35a3f2c5
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/memory.cpp
@@ -0,0 +1,11 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#include "pch.hpp"
+#include "memory.h"
+#include "memory.tmh"
+
+// for wil::make_unique_nothrow
+void*
+operator new(size_t s, std::nothrow_t const&)
+{
+ return operator new(s);
+} \ No newline at end of file
diff --git a/network/netadaptercx/netvadapterlibrary/code/memory.h b/network/netadaptercx/netvadapterlibrary/code/memory.h
new file mode 100644
index 00000000..1200c72c
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/memory.h
@@ -0,0 +1,4 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#pragma once
+
+#define MAX_RX_BUFFER_SIZE 65535
diff --git a/network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h b/network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h
new file mode 100644
index 00000000..b5e34518
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/net/netpacketlibrary.h
@@ -0,0 +1,85 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#pragma once
+
+#include <net/virtualaddress.h>
+
+//
+// Following are some helper APIs for common ring manipulations.
+// They are all implemented using iterator
+//
+
+inline
+SIZE_T
+GetTxPacketDataLength(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ SIZE_T length = 0;
+
+ for (NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(Iterator);
+ NetFragmentIteratorHasAny(&fi);
+ NetFragmentIteratorAdvance(&fi))
+ {
+ NET_FRAGMENT* fragment = NetFragmentIteratorGetFragment(&fi);
+ length += (SIZE_T)fragment->ValidLength;
+ }
+
+ return length;
+}
+
+inline
+VOID
+CompleteTxPacketsBatch(
+ _In_ NET_RING_COLLECTION const* Rings,
+ _In_ UINT32 BatchSize
+)
+{
+ UINT32 packetCount = 0;
+
+ NET_RING_PACKET_ITERATOR pi = NetRingGetDrainPackets(Rings);
+
+ while (NetPacketIteratorHasAny(&pi))
+ {
+ NET_PACKET* packet = NetPacketIteratorGetPacket(&pi);
+
+ // this function uses Scratch field as the bit for testing completion
+ if (!packet->Scratch)
+ {
+ break;
+ }
+
+ packetCount++;
+
+ NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(&pi);
+ NetFragmentIteratorAdvanceToTheEnd(&fi);
+
+ NetPacketIteratorAdvance(&pi);
+
+ if (packetCount >= BatchSize)
+ {
+ NetPacketIteratorSet(&pi);
+ Rings->Rings[NetRingTypeFragment]->BeginIndex = NetFragmentIteratorGetIndex(&fi);
+ }
+ }
+}
+
+inline
+void
+CancelRxPackets(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING_PACKET_ITERATOR pi = NetRingGetAllPackets(Rings);
+
+ for (; NetPacketIteratorHasAny(&pi); NetPacketIteratorAdvance(&pi))
+ {
+ NetPacketIteratorGetPacket(&pi)->Ignore = 1;
+ }
+
+ NetPacketIteratorSet(&pi);
+
+ NET_RING_FRAGMENT_ITERATOR fi = NetRingGetAllFragments(Rings);
+ NetFragmentIteratorAdvanceToTheEnd(&fi);
+ NetFragmentIteratorSet(&fi);
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h b/network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h
new file mode 100644
index 00000000..fb010751
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/net/netringiterator.h
@@ -0,0 +1,281 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#pragma once
+
+#include <net/ringcollection.h>
+
+typedef struct _NET_RING_ITERATOR
+{
+
+ NET_RING_COLLECTION const*
+ Rings;
+
+ UINT32* const
+ IndexToSet;
+
+ UINT32
+ Index;
+
+ UINT32 const
+ End;
+
+} NET_RING_ITERATOR;
+
+typedef struct _NET_RING_PACKET_ITERATOR
+{
+
+ NET_RING_ITERATOR
+ Iterator;
+
+} NET_RING_PACKET_ITERATOR;
+
+typedef struct _NET_RING_FRAGMENT_ITERATOR
+{
+
+ NET_RING_ITERATOR
+ Iterator;
+
+} NET_RING_FRAGMENT_ITERATOR;
+
+
+inline
+NET_RING_PACKET_ITERATOR
+NetRingGetPostPackets(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING* ring = Rings->Rings[NetRingTypePacket];
+ NET_RING_PACKET_ITERATOR iterator = {
+ Rings, &ring->NextIndex, ring->NextIndex, ring->EndIndex,
+ };
+
+ return iterator;
+}
+
+inline
+NET_RING_PACKET_ITERATOR
+NetRingGetDrainPackets(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING* ring = Rings->Rings[NetRingTypePacket];
+ NET_RING_PACKET_ITERATOR iterator = {
+ Rings, &ring->BeginIndex, ring->BeginIndex, ring->NextIndex,
+ };
+
+ return iterator;
+}
+
+inline
+NET_RING_PACKET_ITERATOR
+NetRingGetAllPackets(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING* ring = Rings->Rings[NetRingTypePacket];
+ NET_RING_PACKET_ITERATOR iterator = {
+ Rings, &ring->BeginIndex, ring->BeginIndex, ring->EndIndex,
+ };
+
+ return iterator;
+}
+
+inline
+NET_PACKET*
+NetPacketIteratorGetPacket(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ return NetRingGetPacketAtIndex(
+ Iterator->Iterator.Rings->Rings[NetRingTypePacket],
+ Iterator->Iterator.Index);
+}
+
+inline
+UINT32
+NetPacketIteratorGetIndex(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ return Iterator->Iterator.Index;
+}
+
+inline
+BOOLEAN
+NetPacketIteratorHasAny(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ return Iterator->Iterator.Index != Iterator->Iterator.End;
+}
+
+inline
+UINT32
+NetPacketIteratorGetCount(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ NET_RING const* ring = Iterator->Iterator.Rings->Rings[NetRingTypePacket];
+
+ return (Iterator->Iterator.End - Iterator->Iterator.Index) & ring->ElementIndexMask;
+}
+
+inline
+void
+NetPacketIteratorAdvance(
+ _In_ NET_RING_PACKET_ITERATOR* Iterator
+)
+{
+ Iterator->Iterator.Index = NetRingIncrementIndex(
+ Iterator->Iterator.Rings->Rings[NetRingTypePacket],
+ Iterator->Iterator.Index);
+}
+
+inline
+void
+NetPacketIteratorAdvanceToTheEnd(
+ _In_ NET_RING_PACKET_ITERATOR* Iterator
+)
+{
+ Iterator->Iterator.Index = Iterator->Iterator.End;
+}
+
+inline
+void
+NetPacketIteratorSet(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ *Iterator->Iterator.IndexToSet
+ = Iterator->Iterator.Index;
+}
+
+
+inline
+NET_RING_FRAGMENT_ITERATOR
+NetPacketIteratorGetFragments(
+ _In_ NET_RING_PACKET_ITERATOR const* Iterator
+)
+{
+ NET_RING const* ring = Iterator->Iterator.Rings->Rings[NetRingTypeFragment];
+ NET_PACKET const* packet = NetPacketIteratorGetPacket(Iterator);
+ UINT32 const end = NetRingIncrementIndex(ring,
+ packet->FragmentIndex + packet->FragmentCount - 1);
+ NET_RING_FRAGMENT_ITERATOR iterator = {
+ Iterator->Iterator.Rings, NULL, packet->FragmentIndex, end,
+ };
+
+ return iterator;
+}
+
+inline
+NET_RING_FRAGMENT_ITERATOR
+NetRingGetPostFragments(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING* ring = Rings->Rings[NetRingTypeFragment];
+ NET_RING_FRAGMENT_ITERATOR iterator = {
+ Rings, &ring->NextIndex, ring->NextIndex, ring->EndIndex,
+ };
+
+ return iterator;
+}
+
+inline
+NET_RING_FRAGMENT_ITERATOR
+NetRingGetDrainFragments(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING* ring = Rings->Rings[NetRingTypeFragment];
+ NET_RING_FRAGMENT_ITERATOR iterator = {
+ Rings, &ring->BeginIndex, ring->BeginIndex, ring->NextIndex,
+ };
+
+ return iterator;
+}
+
+inline
+NET_RING_FRAGMENT_ITERATOR
+NetRingGetAllFragments(
+ _In_ NET_RING_COLLECTION const* Rings
+)
+{
+ NET_RING* ring = Rings->Rings[NetRingTypeFragment];
+ NET_RING_FRAGMENT_ITERATOR iterator = {
+ Rings, &ring->BeginIndex, ring->BeginIndex, ring->EndIndex,
+ };
+
+ return iterator;
+}
+
+inline
+NET_FRAGMENT*
+NetFragmentIteratorGetFragment(
+ _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator
+)
+{
+ return NetRingGetFragmentAtIndex(
+ Iterator->Iterator.Rings->Rings[NetRingTypeFragment],
+ Iterator->Iterator.Index);
+}
+
+inline
+UINT32
+NetFragmentIteratorGetIndex(
+ _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator
+)
+{
+ return Iterator->Iterator.Index;
+}
+
+inline
+BOOLEAN
+NetFragmentIteratorHasAny(
+ _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator
+)
+{
+ return Iterator->Iterator.Index != Iterator->Iterator.End;
+}
+
+inline
+UINT32
+NetFragmentIteratorGetCount(
+ _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator
+)
+{
+ NET_RING const* ring = Iterator->Iterator.Rings->Rings[NetRingTypeFragment];
+
+ return (Iterator->Iterator.End - Iterator->Iterator.Index) & ring->ElementIndexMask;
+}
+
+inline
+void
+NetFragmentIteratorAdvance(
+ _In_ NET_RING_FRAGMENT_ITERATOR* Iterator
+)
+{
+ Iterator->Iterator.Index = NetRingIncrementIndex(
+ Iterator->Iterator.Rings->Rings[NetRingTypeFragment],
+ Iterator->Iterator.Index);
+}
+
+inline
+void
+NetFragmentIteratorAdvanceToTheEnd(
+ _In_ NET_RING_FRAGMENT_ITERATOR* Iterator
+)
+{
+ Iterator->Iterator.Index = Iterator->Iterator.End;
+}
+
+inline
+void
+NetFragmentIteratorSet(
+ _In_ NET_RING_FRAGMENT_ITERATOR const* Iterator
+)
+{
+ *(Iterator->Iterator.IndexToSet)
+ = Iterator->Iterator.Index;
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h b/network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h
new file mode 100644
index 00000000..a6a019e4
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/net/umxfilter.h
@@ -0,0 +1,268 @@
+/*++
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Module Name:
+
+ xfilter.h
+
+Abstract:
+
+ Header file for the address filtering library for NDIS MAC's.
+
+Author:
+
+Environment:
+
+Notes:
+
+ None.
+
+Revision History:
+
+--*/
+
+#ifndef _X_FILTER_DEFS_
+#define _X_FILTER_DEFS_
+
+#pragma once
+
+#define ETH_LENGTH_OF_ADDRESS 6
+
+
+//
+// ZZZ This is a little-endian specific check.
+//
+#define ETH_IS_MULTICAST(Address) \
+ (BOOLEAN)(((PUCHAR)(Address))[0] & ((UCHAR)0x01))
+
+
+//
+// Check whether an address is broadcast.
+//
+#define ETH_IS_BROADCAST(Address) \
+ ((((PUCHAR)(Address))[0] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[1] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[2] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[3] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[4] == ((UCHAR)0xff)) && (((PUCHAR)(Address))[5] == ((UCHAR)0xff)))
+
+
+//
+// This macro will compare network addresses.
+//
+// A - Is a network address.
+//
+// B - Is a network address.
+//
+// Result - The result of comparing two network address.
+//
+// Result < 0 Implies the B address is greater.
+// Result > 0 Implies the A element is greater.
+// Result = 0 Implies equality.
+//
+// Note that this is an arbitrary ordering. There is not
+// defined relation on network addresses. This is ad-hoc!
+//
+//
+#define ETH_COMPARE_NETWORK_ADDRESSES(_A, _B, _Result) \
+{ \
+ if (*(ULONG UNALIGNED *)&(_A)[2] > \
+ *(ULONG UNALIGNED *)&(_B)[2]) \
+ { \
+ *(_Result) = 1; \
+ } \
+ else if (*(ULONG UNALIGNED *)&(_A)[2] < \
+ *(ULONG UNALIGNED *)&(_B)[2]) \
+ { \
+ *(_Result) = (UINT)-1; \
+ } \
+ else if (*(USHORT UNALIGNED *)(_A) > \
+ *(USHORT UNALIGNED *)(_B)) \
+ { \
+ *(_Result) = 1; \
+ } \
+ else if (*(USHORT UNALIGNED *)(_A) < \
+ *(USHORT UNALIGNED *)(_B)) \
+ { \
+ *(_Result) = (UINT)-1; \
+ } \
+ else \
+ { \
+ *(_Result) = 0; \
+ } \
+}
+
+//
+// This macro will compare network addresses.
+//
+// A - Is a network address.
+//
+// B - Is a network address.
+//
+// Result - The result of comparing two network address.
+//
+// Result != 0 Implies inequality.
+// Result == 0 Implies equality.
+//
+//
+#define ETH_COMPARE_NETWORK_ADDRESSES_EQ(_A,_B, _Result) \
+{ \
+ if ((*(ULONG UNALIGNED *)&(_A)[2] == \
+ *(ULONG UNALIGNED *)&(_B)[2]) && \
+ (*(USHORT UNALIGNED *)(_A) == \
+ *(USHORT UNALIGNED *)(_B))) \
+ { \
+ *(_Result) = 0; \
+ } \
+ else \
+ { \
+ *(_Result) = 1; \
+ } \
+}
+
+
+//
+// This macro is used to copy from one network address to
+// another.
+//
+#define ETH_COPY_NETWORK_ADDRESS(_D, _S) \
+{ \
+ *((ULONG UNALIGNED *)(_D)) = *((ULONG UNALIGNED *)(_S)); \
+ *((USHORT UNALIGNED *)((UCHAR *)(_D)+4)) = *((USHORT UNALIGNED *)((UCHAR *)(_S)+4)); \
+}
+
+#define TR_LENGTH_OF_FUNCTIONAL 4
+#define TR_LENGTH_OF_ADDRESS 6
+
+
+//
+// Only the low 32 bits of the functional/group address
+// are needed since the upper 16 bits is always c0-00.
+//
+typedef ULONG TR_FUNCTIONAL_ADDRESS;
+typedef ULONG TR_GROUP_ADDRESS;
+
+
+#define TR_IS_NOT_DIRECTED(_Address, _Result) \
+{ \
+ *(_Result) = (BOOLEAN)((_Address)[0] & 0x80); \
+}
+
+#define TR_IS_FUNCTIONAL(_Address, _Result) \
+{ \
+ *(_Result) = (BOOLEAN)(((_Address)[0] & 0x80) && \
+ !((_Address)[2] & 0x80)); \
+}
+
+//
+//
+#define TR_IS_GROUP(_Address, _Result) \
+{ \
+ *(_Result) = (BOOLEAN)((_Address)[0] & (_Address)[2] & 0x80); \
+}
+
+//
+//
+#define TR_IS_SOURCE_ROUTING(_Address, _Result) \
+{ \
+ *(_Result) = (BOOLEAN)((_Address)[0] & 0x80); \
+}
+
+//
+// Check for NDIS_PACKET_TYPE_MAC_FRAME
+//
+#define TR_IS_MAC_FRAME(_PacketHeader) ((((PUCHAR)_PacketHeader)[1] & 0xFC) == 0)
+
+
+//
+// Check whether an address is broadcast. This is a little-endian check.
+//
+#define TR_IS_BROADCAST(_Address, _Result) \
+{ \
+ *(_Result) = (BOOLEAN)(((*(UNALIGNED USHORT *)&(_Address)[0] == 0xFFFF) || \
+ (*(UNALIGNED USHORT *)&(_Address)[0] == 0x00C0)) && \
+ (*(UNALIGNED ULONG *)&(_Address)[2] == 0xFFFFFFFF));\
+}
+
+
+//
+// This macro will compare network addresses.
+//
+// A - Is a network address.
+//
+// B - Is a network address.
+//
+// Result - The result of comparing two network address.
+//
+// Result < 0 Implies the B address is greater.
+// Result > 0 Implies the A element is greater.
+// Result = 0 Implies equality.
+//
+// Note that this is an arbitrary ordering. There is not
+// defined relation on network addresses. This is ad-hoc!
+//
+//
+#define TR_COMPARE_NETWORK_ADDRESSES(_A, _B, _Result) \
+{ \
+ if (*(ULONG UNALIGNED *)&(_A)[2] > \
+ *(ULONG UNALIGNED *)&(_B)[2]) \
+ { \
+ *(_Result) = 1; \
+ } \
+ else if (*(ULONG UNALIGNED *)&(_A)[2] < \
+ *(ULONG UNALIGNED *)&(_B)[2]) \
+ { \
+ *(_Result) = (UINT)-1; \
+ } \
+ else if (*(USHORT UNALIGNED *)(_A) > \
+ *(USHORT UNALIGNED *)(_B)) \
+ { \
+ *(_Result) = 1; \
+ } \
+ else if (*(USHORT UNALIGNED *)(_A) < \
+ *(USHORT UNALIGNED *)(_B)) \
+ { \
+ *(_Result) = (UINT)-1; \
+ } \
+ else \
+ { \
+ *(_Result) = 0; \
+ } \
+}
+
+//
+// This macro will compare network addresses.
+//
+// A - Is a network address.
+//
+// B - Is a network address.
+//
+// Result - The result of comparing two network address.
+//
+// Result != 0 Implies inequality.
+// Result == 0 Implies equality.
+//
+//
+#define TR_COMPARE_NETWORK_ADDRESSES_EQ(_A, _B, _Result) \
+{ \
+ if ((*(ULONG UNALIGNED *)&(_A)[2] == *(ULONG UNALIGNED *)&(_B)[2]) && \
+ (*(USHORT UNALIGNED *)&(_A)[0] == *(USHORT UNALIGNED *)&(_B)[0])) \
+ { \
+ *(_Result) = 0; \
+ } \
+ else \
+ { \
+ *(_Result) = 1; \
+ } \
+}
+
+
+//
+// This macro is used to copy from one network address to
+// another.
+//
+#define TR_COPY_NETWORK_ADDRESS(_D, _S) \
+{ \
+ *((ULONG UNALIGNED *)(_D)) = *((ULONG UNALIGNED *)(_S)); \
+ *((USHORT UNALIGNED *)((UCHAR *)(_D)+4)) = \
+ *((USHORT UNALIGNED *)((UCHAR *)(_S)+4)); \
+}
+
+#endif // _X_FILTER_DEFS_
diff --git a/network/netadaptercx/netvadapterlibrary/code/pch.hpp b/network/netadaptercx/netvadapterlibrary/code/pch.hpp
new file mode 100644
index 00000000..cf2f0f22
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/pch.hpp
@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+#pragma once
+
+#include <initguid.h>
+
+#ifndef _KERNEL_MODE
+// This is a user-mode driver
+#include <windows.h>
+
+#else
+// This is a kernel-mode driver
+#include <ntddk.h>
+#define NTSTRSAFE_LIB
+#include <ntstrsafe.h>
+#endif
+
+// This is a common WDF header (for both KMDF and UMDF)
+#include <wdf.h>
+
+#include <netadaptercx.h>
+//#include <wdftriage.h>
+#include "net/netringiterator.h"
+#include "net/netpacketlibrary.h"
+#include <net/rsc.h>
+#include <net/gso.h>
+#include <net/checksum.h>
+#include <net/databuffer.h>
+#include <net/returncontext.h>
+
+#include "enlthreads.h"
+#include "enl.h"
+
+#include "trace.h"
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/queue.cpp b/network/netadaptercx/netvadapterlibrary/code/queue.cpp
new file mode 100644
index 00000000..667d59f7
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/queue.cpp
@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+#include "pch.hpp"
+
+#include <net/ringcollection.h>
+
+#include "queue.h"
+
+NetvQueue::NetvQueue(
+ NETPACKETQUEUE Handle,
+ NetvAdapter & Adapter,
+ NET_RING_COLLECTION const * Rings
+)
+ : m_handle{Handle}
+ , m_adapter{Adapter}
+ , m_rings{Rings}
+{
+}
+
+NET_RING *
+NetvQueue::GetPacketRing(
+ void
+)
+{
+ return NetRingCollectionGetPacketRing(m_rings);
+}
+
+NET_RING *
+NetvQueue::GetFragmentRing(
+ void
+)
+{
+ return NetRingCollectionGetFragmentRing(m_rings);
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/queue.h b/network/netadaptercx/netvadapterlibrary/code/queue.h
new file mode 100644
index 00000000..becb72ce
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/queue.h
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+#pragma once
+
+class NetvAdapter;
+struct ENLP_QUEUE;
+
+class NetvQueue
+{
+
+public:
+
+ NetvQueue(
+ NETPACKETQUEUE Handle,
+ NetvAdapter & Adapter,
+ NET_RING_COLLECTION const * Rings
+ );
+
+ NET_RING *
+ GetPacketRing(
+ void
+ );
+
+ NET_RING *
+ GetFragmentRing(
+ void
+ );
+
+ NETPACKETQUEUE const
+ m_handle{WDF_NO_HANDLE};
+
+ NetvAdapter &
+ m_adapter;
+
+ NET_RING_COLLECTION const *
+ m_rings;
+
+ ENLP_QUEUE *
+ EnlQueueHandle;
+
+};
diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h
new file mode 100644
index 00000000..0bbedadb
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KCriticalRegion.h
@@ -0,0 +1,67 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#pragma once
+
+#include <KMacros.h>
+
+class KCriticalRegion
+{
+public:
+
+ PAGED KCriticalRegion() : m_Entered(false) { }
+
+ PAGED ~KCriticalRegion() { if (m_Entered) Leave(); }
+
+ KCriticalRegion(KCriticalRegion &) = delete;
+ KCriticalRegion &operator=(KCriticalRegion &) = delete;
+
+ PAGED void Enter()
+ {
+ ASSERT(m_Entered == false);
+ UnbalancedEnter();
+ m_Entered = true;
+ }
+
+ PAGED void Leave()
+ {
+ ASSERT(m_Entered == true);
+ m_Entered = false;
+ UnbalancedLeave();
+ }
+
+ static PAGED void UnbalancedEnter()
+ {
+#if _KERNEL_MODE
+ KeEnterCriticalRegion();
+#endif
+ }
+
+ static PAGED void UnbalancedLeave()
+ {
+#if _KERNEL_MODE
+ KeLeaveCriticalRegion();
+#endif
+ }
+
+private:
+
+ bool m_Entered;
+};
+
+struct KDefaultRegion
+{
+ void Enter() { }
+ void Leave() { }
+};
+
+struct KIrqlRegion
+{
+ KIrqlRegion() { }
+ ~KIrqlRegion() { }
+
+ void Enter() { }
+ void Leave() { }
+
+ KIRQL m_OldIrql;
+};
+
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h
new file mode 100644
index 00000000..d398f137
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KLockHolder.h
@@ -0,0 +1,114 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#pragma once
+
+#include <KCriticalRegion.h>
+#include <KPushLock.h>
+
+class KRTL_CLASS KLockHolder
+{
+private:
+
+ enum { Unlocked, Shared, Exclusive } m_State;
+
+public:
+
+ PAGED KLockHolder(KPushLockBase &lock) : m_Lock(lock), m_State(Unlocked) { }
+ PAGED ~KLockHolder()
+ {
+ switch (m_State)
+ {
+ case Shared:
+ ReleaseShared();
+ break;
+ case Exclusive:
+ ReleaseExclusive();
+ break;
+ }
+ }
+
+ KLockHolder(KLockHolder &) = delete;
+ KLockHolder &operator=(KLockHolder &) = delete;
+
+ _IRQL_requires_(PASSIVE_LEVEL)
+ PAGED void AcquireShared()
+ {
+ m_Region.Enter();
+ ASSERT(m_State == Unlocked);
+ m_Lock.AcquireShared();
+ m_State = Shared;
+ }
+
+ _IRQL_requires_(PASSIVE_LEVEL)
+ PAGED void ReleaseShared()
+ {
+ ASSERT(m_State == Shared);
+ m_Lock.ReleaseShared();
+ m_State = Unlocked;
+ m_Region.Leave();
+ }
+
+ _IRQL_requires_(PASSIVE_LEVEL)
+ PAGED void AcquireExclusive()
+ {
+ m_Region.Enter();
+ ASSERT(m_State == Unlocked);
+ m_Lock.AcquireExclusive();
+ m_State = Exclusive;
+ }
+
+ _IRQL_requires_(PASSIVE_LEVEL)
+ PAGED void ReleaseExclusive()
+ {
+ ASSERT(m_State == Exclusive);
+ m_Lock.ReleaseExclusive();
+ m_State = Unlocked;
+ m_Region.Leave();
+ }
+
+private:
+
+ KPushLockBase &m_Lock;
+ KCriticalRegion m_Region;
+};
+
+class KRTL_CLASS KLockThisShared : protected KLockHolder
+{
+public:
+
+ PAGED KLockThisShared(KPushLockBase &lock) : KLockHolder(lock)
+ {
+ AcquireShared();
+ }
+
+ PAGED void Acquire()
+ {
+ AcquireShared();
+ }
+
+ PAGED void Release()
+ {
+ ReleaseShared();
+ }
+};
+
+class KRTL_CLASS KLockThisExclusive : protected KLockHolder
+{
+public:
+
+ PAGED KLockThisExclusive(KPushLockBase &lock) : KLockHolder(lock)
+ {
+ AcquireExclusive();
+ }
+
+ PAGED void Acquire()
+ {
+ AcquireExclusive();
+ }
+
+ PAGED void Release()
+ {
+ ReleaseExclusive();
+ }
+};
+
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h
new file mode 100644
index 00000000..631c08bf
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KMacros.h
@@ -0,0 +1,84 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#pragma once
+
+#include <wil/common.h>
+
+#ifdef _KERNEL_MODE
+
+// nullptr_t is normally automatically defined by the CRT headers, but it
+// doesn't get included by kernel code.
+namespace std { typedef decltype(__nullptr) nullptr_t; }
+using ::std::nullptr_t;
+
+// The stddef.h used for kernel code has the old offsetof macro.
+// Let's use the new one instead.
+#undef offsetof
+#define offsetof(s,m) __builtin_offsetof(s,m)
+
+#endif // _KERNEL_MODE
+
+#define BEGIN_MACRO do {
+#define END_MACRO } while (0)
+
+#ifdef _KERNEL_MODE
+#define CODE_SEG(segment) __declspec(code_seg(segment))
+#else
+#define CODE_SEG(segment)
+#endif
+
+#ifndef KRTL_PAGE_SEGMENT
+# define KRTL_PAGE_SEGMENT "PAGE"
+#endif
+#ifndef KRTL_INIT_SEGMENT
+# define KRTL_INIT_SEGMENT "INIT"
+#endif
+#ifndef KRTL_NONPAGED_SEGMENT
+# define KRTL_NONPAGED_SEGMENT ".text"
+#endif
+
+/// Use on pageable functions.
+#define PAGED CODE_SEG(KRTL_PAGE_SEGMENT) _IRQL_always_function_max_(PASSIVE_LEVEL)
+
+/// Use on pageable functions, where you don't want the SAL IRQL annotation to say PASSIVE_LEVEL.
+#define PAGEDX CODE_SEG(KRTL_PAGE_SEGMENT)
+
+/// Use on code in the INIT segment. (Code is discarded after DriverEntry returns.)
+#define INITCODE CODE_SEG(KRTL_INIT_SEGMENT)
+
+/// Use on code that must always be locked in memory.
+#define NONPAGED CODE_SEG(KRTL_NONPAGED_SEGMENT) _IRQL_requires_max_(DISPATCH_LEVEL)
+
+/// Use on code that must always be locked in memory, where you don't want SAL IRQL annotations.
+#define NONPAGEDX CODE_SEG(KRTL_NONPAGED_SEGMENT)
+
+#ifndef _KERNEL_MODE
+
+#ifndef PAGED_CODE
+#define PAGED_CODE() (void)0
+#endif // PAGED_CODE
+
+#endif // _KERNEL_MODE
+
+/// Use on classes or structs. Class member functions & compiler-generated code
+/// will default to the PAGE segment. You can override any member function with `NONPAGED`.
+#define KRTL_CLASS CODE_SEG(KRTL_PAGE_SEGMENT) __declspec(empty_bases)
+
+/// Use on classes or structs. Class member functions & compiler-generated code
+/// will default to the NONPAGED segment. You can override any member function with `PAGED`.
+#define KRTL_CLASS_DPC_ALLOC __declspec(empty_bases)
+
+enum CallRunMode
+{
+ // This call should complete synchronously on the current thread
+ RunSynchronous,
+ // This call should return immediately, and complete the operation in a background thread
+ RunAsynchronous,
+ // This call can return immediately, OR complete synchronously
+ // (Use this if you're running on a workitem thread already, and you
+ // don't mind if the callee uses your thread to do its work, but you
+ // can tolerate the call completing asynchronously if the callee doesn't
+ // need your thread.)
+ RunAsynchronousButOkayToBlock,
+};
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h
new file mode 100644
index 00000000..c589866c
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KPushLock.h
@@ -0,0 +1,147 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#pragma once
+
+#include <KMacros.h>
+#include <KCriticalRegion.h>
+
+typedef struct _KTHREAD *PKTHREAD;
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+class KPushLockBase
+{
+public:
+
+ KPushLockBase() = default;
+ KPushLockBase(KPushLockBase &) = delete;
+ KPushLockBase & operator=(KPushLockBase &) = delete;
+
+ PAGED
+ void
+ KPushLockBase::AcquireShared()
+ {
+#ifdef _KERNEL_MODE
+ ExAcquirePushLockShared(&m_Lock);
+#else
+ AcquireSRWLockShared(&m_Lock);
+#endif
+ }
+
+ PAGED
+ void
+ KPushLockBase::ReleaseShared()
+ {
+#ifdef _KERNEL_MODE
+ ExReleasePushLockShared(&m_Lock);
+#else
+ ReleaseSRWLockShared(&m_Lock);
+#endif
+ }
+
+ PAGED
+ void
+ KPushLockBase::AcquireExclusive()
+ {
+#ifdef _KERNEL_MODE
+ ExAcquirePushLockExclusive(&m_Lock);
+#if DBG
+ m_ExclusiveOwner = KeGetCurrentThread();
+#endif
+#else
+ AcquireSRWLockExclusive(&m_Lock);
+#endif
+
+ }
+
+ PAGED
+ void
+ KPushLockBase::ReleaseExclusive()
+ {
+#if DBG
+ m_ExclusiveOwner = nullptr;
+#endif
+#ifdef _KERNEL_MODE
+ ExReleasePushLockExclusive(&m_Lock);
+#else
+ ReleaseSRWLockExclusive(&m_Lock);
+#endif
+ }
+
+ PAGED
+ void
+ KPushLockBase::AssertLockHeld()
+ {
+#ifdef _KERNEL_MODE
+ WIN_ASSERT(m_ExclusiveOwner == KeGetCurrentThread());
+#endif
+ }
+
+ PAGED
+ void
+ KPushLockBase::AssertLockNotHeld()
+ {
+#if DBG && defined(_KERNEL_MODE)
+ WIN_ASSERT(m_ExclusiveOwner != KeGetCurrentThread());
+#endif
+ }
+
+protected:
+
+ PAGED
+ void
+ KPushLockBase::InitializeInner()
+ {
+#ifdef _KERNEL_MODE
+ ExInitializePushLock(&m_Lock);
+#else
+ InitializeSRWLock(&m_Lock);
+#endif
+#if DBG
+ m_ExclusiveOwner = nullptr;
+#endif
+ }
+
+private:
+
+#ifdef _KERNEL_MODE
+ EX_PUSH_LOCK m_Lock;
+#else
+ SRWLOCK m_Lock;
+#endif
+
+#if DBG
+ PKTHREAD m_ExclusiveOwner;
+#endif
+};
+
+class KPushLock : public KPushLockBase
+{
+public:
+
+ PAGED
+ KPushLock::KPushLock() noexcept
+ {
+ InitializeInner();
+ }
+
+
+
+ PAGED
+ KPushLock::~KPushLock()
+ {
+ AssertLockNotHeld();
+ }
+};
+
+class KPushLockManualConstruct : public KPushLockBase
+{
+public:
+
+ PAGED
+ void
+ KPushLockManualConstruct::Initialize()
+ {
+ InitializeInner();
+ }
+};
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h b/network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h
new file mode 100644
index 00000000..124f416e
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rtl/KWaitEvent.h
@@ -0,0 +1,172 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+#pragma once
+
+#include <KMacros.h>
+
+#if _KERNEL_MODE
+typedef wistd::integral_constant<EVENT_TYPE, SynchronizationEvent> auto_reset_event_t;
+#else
+typedef wistd::integral_constant<bool, false> auto_reset_event_t;
+#endif
+
+#if _KERNEL_MODE
+typedef wistd::integral_constant<EVENT_TYPE, NotificationEvent> manual_reset_event_t;
+#else
+typedef wistd::integral_constant<bool, true> manual_reset_event_t;
+#endif
+
+#define WIN_VERIFY NT_VERIFY
+#define WIN_ASSERT NT_ASSERT
+
+template<typename TEventType>
+class KWaitEventBase
+{
+public:
+
+ KWaitEventBase() = default;
+
+#ifdef _KERNEL_MODE
+ NONPAGED ~KWaitEventBase() = default;
+#else
+ NONPAGED ~KWaitEventBase()
+ {
+ WIN_ASSERT(m_event == nullptr);
+ }
+#endif
+
+ KWaitEventBase(KWaitEventBase &) = delete;
+ KWaitEventBase(KWaitEventBase &&) = delete;
+ KWaitEventBase & operator=(KWaitEventBase &) = delete;
+ KWaitEventBase & operator=(KWaitEventBase &&) = delete;
+
+ _IRQL_requires_max_(DISPATCH_LEVEL) void Set()
+ {
+#if _KERNEL_MODE
+ KeSetEvent(&m_event, 0, false);
+#else
+ WIN_VERIFY(SetEvent(m_event));
+#endif
+ }
+
+ _IRQL_requires_max_(DISPATCH_LEVEL) void Clear()
+ {
+#if _KERNEL_MODE
+ KeClearEvent(&m_event);
+#else
+ WIN_VERIFY(ResetEvent(m_event));
+#endif
+ }
+
+ PAGED void Wait()
+ {
+#if _KERNEL_MODE
+ // Not used at runtime, but might be useful during debugging
+ volatile LARGE_INTEGER SystemTime;
+ KeQuerySystemTime(const_cast<LARGE_INTEGER*>(&SystemTime));
+
+ NTSTATUS NtStatus = KeWaitForSingleObject(
+ &m_event, Executive, KernelMode, FALSE, nullptr);
+ NT_VERIFY(NtStatus == STATUS_SUCCESS);
+#else
+ ULONG r = WaitForSingleObject(m_event, INFINITE);
+ WIN_VERIFY(r == NO_ERROR);
+#endif
+ }
+
+ PAGED bool Test()
+ {
+#if _KERNEL_MODE
+ return !!KeReadStateEvent(&m_event);
+#else
+ ULONG r = WaitForSingleObject(m_event, 0);
+ WIN_VERIFY(r == WAIT_TIMEOUT || r == WAIT_OBJECT_0);
+ return (r == WAIT_OBJECT_0);
+#endif
+ }
+
+ NONPAGED bool TestNP()
+ {
+#if _KERNEL_MODE
+ return !!KeReadStateEvent(&m_event);
+#else
+ ULONG r = WaitForSingleObject(m_event, 0);
+ WIN_VERIFY(r == WAIT_TIMEOUT || r == WAIT_OBJECT_0);
+ return (r == WAIT_OBJECT_0);
+#endif
+ }
+
+protected:
+
+ PAGED void InitializeBase()
+ {
+#if _KERNEL_MODE
+ KeInitializeEvent(&m_event, TEventType(), FALSE);
+#else
+ m_event = CreateEventW(nullptr, TEventType(), false, nullptr);
+ WIN_VERIFY(m_event);
+#endif
+ }
+
+ NONPAGED void CleanupBase()
+ {
+#ifndef _KERNEL_MODE
+ CloseHandle(m_event);
+ m_event = nullptr;
+#endif
+ }
+
+private:
+
+#if _KERNEL_MODE
+ KEVENT m_event;
+#else
+ HANDLE m_event;
+#endif
+
+};
+
+class KWaitEvent : public KWaitEventBase<manual_reset_event_t>
+{
+public:
+
+ PAGED KWaitEvent() noexcept
+ {
+ InitializeBase();
+ }
+
+ NONPAGED ~KWaitEvent()
+ {
+ CleanupBase();
+ }
+};
+
+class KWaitEventManualConstruct : public KWaitEventBase<manual_reset_event_t>
+{
+public:
+
+ PAGED void Initialize()
+ {
+ InitializeBase();
+ }
+
+ PAGED void Cleanup()
+ {
+ CleanupBase();
+ }
+};
+
+
+class KAutoEvent : public KWaitEventBase<auto_reset_event_t>
+{
+public:
+
+ PAGED KAutoEvent() noexcept
+ {
+ InitializeBase();
+ }
+
+ NONPAGED ~KAutoEvent()
+ {
+ CleanupBase();
+ }
+};
diff --git a/network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp b/network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp
new file mode 100644
index 00000000..6eed638c
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rxqueue.cpp
@@ -0,0 +1,156 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+#include "pch.hpp"
+#include "netvadapter.h"
+#include "rxqueue.h"
+#include "memory.h"
+
+static
+void
+CheckForWakeFrame(
+ NetvRxQueue * rx
+)
+{
+ NET_RING_FRAGMENT_ITERATOR fi = NetRingGetAllFragments(rx->m_rings);
+
+ if (! NetFragmentIteratorHasAny(&fi))
+ {
+ return;
+ }
+
+ auto *fragment = NetFragmentIteratorGetFragment(&fi);
+ auto *rxVirtualAddress = NetExtensionGetFragmentVirtualAddress(
+ &rx->VirtualAddressExtension,
+ NetFragmentIteratorGetIndex(&fi));
+
+ auto *fragmentBuffer = reinterpret_cast<unsigned char *>(rxVirtualAddress->VirtualAddress) + fragment->Offset;
+
+ fragment->ValidLength = EnlCopyWakeFrame(
+ NetvEnlMLink[rx->m_adapter.EnlIndex].LinkHandle[0],
+ fragmentBuffer,
+ fragment->Capacity);
+
+ // If there was a pending wake frame mark this fragment as complete, the normal advance code will get to it
+ fragment->Scratch = fragment->ValidLength > 0 ? 1 : 0;
+
+ rx->CheckedWakeFrame = true;
+}
+
+
+NetvRxQueue::NetvRxQueue(
+ NETPACKETQUEUE Handle,
+ NetvAdapter & Adapter
+)
+ : NetvQueue{Handle, Adapter, NetRxQueueGetRingCollection(Handle)}
+{
+ NET_EXTENSION_QUERY extension;
+
+ NET_EXTENSION_QUERY_INIT(
+ &extension,
+ NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_NAME,
+ NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_VERSION_1,
+ NetExtensionTypeFragment);
+
+ NetRxQueueGetExtension(m_handle, &extension, &VirtualAddressExtension);
+
+ NET_EXTENSION_QUERY_INIT(
+ &extension,
+ NET_PACKET_EXTENSION_RSC_NAME,
+ NET_PACKET_EXTENSION_RSC_VERSION_2,
+ NetExtensionTypePacket);
+
+ NetRxQueueGetExtension(m_handle, &extension, &UdpRscExtension);
+
+ NET_EXTENSION_QUERY_INIT(
+ &extension,
+ NET_PACKET_EXTENSION_CHECKSUM_NAME,
+ NET_PACKET_EXTENSION_CHECKSUM_VERSION_1,
+ NetExtensionTypePacket);
+
+ NetRxQueueGetExtension(m_handle, &extension, &RxXSumExtension);
+
+ EnlQueueHandle = EnlCreateQueue(Handle, RX);
+}
+
+_Use_decl_annotations_
+void
+NetvRxQueue::Destroy(
+ void
+)
+{
+ EnlDestroyQueue(EnlQueueHandle, RX);
+}
+
+void
+NetvRxQueue::Start(
+ void
+)
+{
+ auto link = NetvEnlMLink[m_adapter.EnlIndex].LinkHandle[0];
+ auto port = &link->Ports[m_adapter.EnlPortIndex];
+ auto queue = &port->RxQueue[0];
+
+ WDFVERIFY(queue->State == Stopped);
+
+ queue->QueueNext = queue->QueueEnd = 0U;
+
+ EnlIndicateQueueState(EnlQueueHandle, Started);
+}
+
+void
+NetvRxQueue::Stop(
+ void
+)
+{
+ EnlIndicateQueueState(EnlQueueHandle, Stopped);
+}
+
+_Use_decl_annotations_
+void
+NetvRxQueue::Advance(
+ void
+)
+{
+ auto fr = GetFragmentRing();
+ NET_RING_PACKET_ITERATOR pi = NetRingGetAllPackets(m_rings);
+ NET_RING_FRAGMENT_ITERATOR fi = NetRingGetAllFragments(m_rings);
+
+ // Ideally this would run in EvtQueueStart, but at that point the receive buffers are not
+ // attached to the fragment yet
+ if (! CheckedWakeFrame)
+ {
+ CheckForWakeFrame(this);
+ }
+
+ // Move begin index forward for all fragments with Scratch == 1, thus returning them to the OS since we're done processing them.
+ for (; NetFragmentIteratorHasAny(&fi) && NetPacketIteratorHasAny(&pi); NetPacketIteratorAdvance(&pi), NetFragmentIteratorAdvance(&fi))
+ {
+ NET_FRAGMENT const * fragment = NetFragmentIteratorGetFragment(&fi);
+ if (! fragment->Scratch)
+ {
+ break;
+ }
+ }
+
+ NetFragmentIteratorSet(&fi);
+ NetPacketIteratorSet(&pi);
+ EnlRingDoorBell(EnlQueueHandle, fr->EndIndex);
+}
+
+_Use_decl_annotations_
+void
+NetvRxQueue::Cancel(
+ void
+)
+{
+ CancelRxPackets(m_rings);
+}
+
+_Use_decl_annotations_
+void
+NetvRxQueue::SetNotify(
+ bool NotificationEnabled
+)
+{
+ EnlArmInterrupt(EnlQueueHandle, NotificationEnabled);
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/rxqueue.h b/network/netadaptercx/netvadapterlibrary/code/rxqueue.h
new file mode 100644
index 00000000..cf208def
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/rxqueue.h
@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+#pragma once
+#include "queue.h"
+
+class NetvRxQueue final
+ : public NetvQueue
+{
+
+public:
+
+ NetvRxQueue(
+ NETPACKETQUEUE Handle,
+ NetvAdapter & Adapter
+ );
+
+ void
+ Destroy(
+ void
+ );
+
+ void
+ Start(
+ void
+ );
+
+ void
+ Stop(
+ void
+ );
+
+ void
+ Advance(
+ void
+ );
+
+ void
+ Cancel(
+ void
+ );
+
+ void
+ SetNotify(
+ bool Enable
+ );
+
+ ENLP_QUEUE * EnlQueueHandle;
+
+ NET_EXTENSION VirtualAddressExtension;
+ NET_EXTENSION UdpRscExtension;
+ NET_EXTENSION RxXSumExtension;
+ NET_EXTENSION NetMemoryExtension;
+ NET_EXTENSION NetMemoryReturnContextExtensionIn;
+
+ bool CheckedWakeFrame = false;
+};
+
+WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(NetvRxQueue, NetvRxQueueGetContext);
diff --git a/network/netadaptercx/netvadapterlibrary/code/trace.h b/network/netadaptercx/netvadapterlibrary/code/trace.h
new file mode 100644
index 00000000..dc2e28e4
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/trace.h
@@ -0,0 +1,96 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#pragma once
+
+#include <initguid.h>
+
+// 5EC34F87-7705-4B5B-B5F6-1F3E4665A31E
+#define WPP_CONTROL_GUIDS \
+ WPP_DEFINE_CONTROL_GUID( \
+ NetvadapterTraceGuid, \
+ (5EC34F87,7705,4B5B,B5F6,1F3E4665A31E), \
+ WPP_DEFINE_BIT(FLAG_DRIVER) \
+ )
+
+#define WPP_FLAG_LEVEL_LOGGER(flag, level) \
+ WPP_LEVEL_LOGGER(flag)
+
+#define WPP_FLAG_LEVEL_ENABLED(flag, level) \
+ (WPP_LEVEL_ENABLED(flag) && WPP_CONTROL(WPP_BIT_ ## flag).Level >= level)
+
+#define WPP_LEVEL_FLAGS_LOGGER(lvl,flags) \
+ WPP_LEVEL_LOGGER(flags)
+
+#define WPP_LEVEL_FLAGS_ENABLED(lvl, flags) \
+ (WPP_LEVEL_ENABLED(flags) && WPP_CONTROL(WPP_BIT_ ## flags).Level >= lvl)
+
+// begin_wpp config
+// USEPREFIX (LogInformation, "%!FUNC! ->");
+// LogVerbose{LEVEL=TRACE_LEVEL_VERBOSE}(FLAGS, MSG, ...);
+// end_wpp
+
+// begin_wpp config
+// USEPREFIX (LogInformation, "%!FUNC! ->");
+// LogWarning{LEVEL=TRACE_LEVEL_WARNING}(FLAGS, MSG, ...);
+// end_wpp
+
+// begin_wpp config
+// USEPREFIX (LogInformation, "%!FUNC! ->");
+// LogError{LEVEL=TRACE_LEVEL_ERROR}(FLAGS, MSG, ...);
+// end_wpp
+
+// begin_wpp config
+// USEPREFIX (LogInformation, "%!FUNC! ->");
+// LogInformation{LEVEL=TRACE_LEVEL_INFORMATION}(FLAGS, MSG, ...);
+// end_wpp
+
+//
+// WPP orders static parameters before dynamic parameters. To support the Trace function
+// defined below which sets FLAGS=MYDRIVER_ALL_INFO, a custom macro must be defined to
+// reorder the arguments to what the .tpl configuration file expects.
+//
+#define WPP_RECORDER_FLAGS_LEVEL_ARGS(flags, lvl) \
+ WPP_RECORDER_LEVEL_FLAGS_ARGS(lvl, flags)
+
+#define WPP_RECORDER_FLAGS_LEVEL_FILTER(flags, lvl) \
+ WPP_RECORDER_LEVEL_FLAGS_FILTER(lvl, flags)
+
+// begin_wpp config
+// USEPREFIX (RETURN_IF_NOT_STATUS_SUCCESS, "%!STATUS! %!FUNC! ->%!s!", nt__wpp, #NTSTATUS);
+// FUNC RETURN_IF_NOT_STATUS_SUCCESS{FLAG=FLAG_DRIVER,LEVEL=TRACE_LEVEL_ERROR}(NTSTATUS);
+// end_wpp
+
+#define WPP_FLAG_LEVEL_NTSTATUS_PRE(flag, level, ntstatus) do { NTSTATUS nt__wpp = (ntstatus); if (STATUS_SUCCESS != nt__wpp) {
+#define WPP_FLAG_LEVEL_NTSTATUS_POST(flag, level, ntstatus); return nt__wpp; } } while (0)
+#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_FILTER(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag)
+#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_ARGS(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag)
+
+// begin_wpp config
+// USEPREFIX (RETURN_NTSTATUS_IF, "%!STATUS! %!FUNC! ->%!s!", nt__wpp, #CONDITION);
+// FUNC RETURN_NTSTATUS_IF{FLAG=FLAG_DRIVER,LEVEL=TRACE_LEVEL_ERROR}(NTSTATUS, CONDITION);
+// end_wpp
+
+#define WPP_FLAG_LEVEL_NTSTATUS_CONDITION_PRE(flag, level, ntstatus, condition) if (condition) { NTSTATUS nt__wpp = (ntstatus);
+#define WPP_FLAG_LEVEL_NTSTATUS_CONDITION_POST(flag, level, ntstatus, condition); return nt__wpp; }
+#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_CONDITION_FILTER(flag, level, ntstatus, condition) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag)
+#define WPP_RECORDER_FLAG_LEVEL_NTSTATUS_CONDITION_ARGS(flag, level, ntstatus, condition) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag)
+
+// begin_wpp config
+// USEPREFIX (RETURN_FAILED_NTSTATUS_MSG, "%!STATUS! %!FUNC! ->", nt__wpp);
+// FUNC RETURN_FAILED_NTSTATUS_MSG{FLAG=FLAG_DRIVER,FAILEDLEVEL=TRACE_LEVEL_ERROR}(NTSTATUS, MSG, ...);
+// end_wpp
+
+#define WPP_FLAG_FAILEDLEVEL_NTSTATUS_PRE(flag, level, ntstatus); do { NTSTATUS nt__wpp = (ntstatus);
+#define WPP_FLAG_FAILEDLEVEL_NTSTATUS_POST(flag, level, ntstatus); return nt__wpp; } while (0)
+#define WPP_RECORDER_FLAG_FAILEDLEVEL_NTSTATUS_FILTER(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag)
+#define WPP_RECORDER_FLAG_FAILEDLEVEL_NTSTATUS_ARGS(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag)
+
+// begin_wpp config
+// USEPREFIX (RETURN_STATUS_SUCCESS, "%!STATUS! %!FUNC!", STATUS_SUCCESS);
+// FUNC RETURN_STATUS_SUCCESS{FLAG=FLAG_DRIVER,SUCCESSLEVEL=TRACE_LEVEL_INFORMATION,NTSTATUS=STATUS_SUCCESS}();
+// end_wpp
+
+#define WPP_FLAG_SUCCESSLEVEL_NTSTATUS_POST(flag, level, ntstatus); return (ntstatus);
+#define WPP_RECORDER_FLAG_SUCCESSLEVEL_NTSTATUS_FILTER(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_FILTER(level, flag)
+#define WPP_RECORDER_FLAG_SUCCESSLEVEL_NTSTATUS_ARGS(flag, level, ntstatus) WPP_RECORDER_LEVEL_FLAGS_ARGS(level, flag)
+
diff --git a/network/netadaptercx/netvadapterlibrary/code/txqueue.cpp b/network/netadaptercx/netvadapterlibrary/code/txqueue.cpp
new file mode 100644
index 00000000..b67692b5
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/txqueue.cpp
@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+#include "pch.hpp"
+#include "netvadapter.h"
+#include "txqueue.h"
+
+NetvTxQueue::NetvTxQueue(
+ NETPACKETQUEUE Handle,
+ NetvAdapter & Adapter
+) noexcept
+ : NetvQueue{Handle, Adapter, NetTxQueueGetRingCollection(Handle)}
+{
+ NET_EXTENSION_QUERY extension;
+ NET_EXTENSION_QUERY_INIT(
+ &extension,
+ NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_NAME,
+ NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_VERSION_1,
+ NetExtensionTypeFragment);
+
+ NetTxQueueGetExtension(m_handle, &extension, &VirtualAddressExtension);
+
+ NET_EXTENSION_QUERY_INIT(
+ &extension,
+ NET_PACKET_EXTENSION_GSO_NAME,
+ NET_PACKET_EXTENSION_GSO_VERSION_1,
+ NetExtensionTypePacket);
+
+ NetTxQueueGetExtension(m_handle, &extension, &UsoExtension);
+
+ EnlQueueHandle = EnlCreateQueue(Handle, TX);
+}
+
+_Use_decl_annotations_
+void
+NetvTxQueue::Destroy(
+ void
+)
+{
+ EnlDestroyQueue(EnlQueueHandle, TX);
+}
+
+void
+NetvTxQueue::Start(
+ void
+)
+{
+ auto link = NetvEnlMLink[m_adapter.EnlIndex].LinkHandle[0];
+ auto port = &link->Ports[m_adapter.EnlPortIndex];
+ auto queue = &port->TxQueue[0];
+
+ NT_ASSERT(queue->State == Stopped);
+
+ queue->QueueNext = queue->QueueEnd = 0U;
+
+ EnlIndicateQueueState(EnlQueueHandle, Started);
+}
+
+void
+NetvTxQueue::Stop(
+ void
+)
+{
+ EnlIndicateQueueState(EnlQueueHandle, Stopped);
+}
+
+_Use_decl_annotations_
+void
+NetvTxQueue::Advance(
+ void
+)
+{
+ auto pr = GetPacketRing();
+ auto pi = NetRingGetAllPackets(m_rings);
+
+ // drain Tx packets
+ for (; NetPacketIteratorHasAny(&pi); NetPacketIteratorAdvance(&pi))
+ {
+ auto packet = NetPacketIteratorGetPacket(&pi);
+ if (! packet->Scratch)
+ {
+ break;
+ }
+
+ auto fi = NetPacketIteratorGetFragments(&pi);
+ for (; NetFragmentIteratorHasAny(&fi); NetFragmentIteratorAdvance(&fi))
+ {
+ continue;
+ }
+
+ m_rings->Rings[NetRingTypeFragment]->BeginIndex =
+ NetFragmentIteratorGetIndex(&fi);
+ }
+
+ NetPacketIteratorSet(&pi);
+
+ // post Tx packets
+ EnlRingDoorBell(EnlQueueHandle, pr->EndIndex);
+}
+
+_Use_decl_annotations_
+void
+NetvTxQueue::Cancel(
+ void
+)
+{
+ auto ringBuffer = GetPacketRing();
+
+ EnlRingDoorBell(EnlQueueHandle, ringBuffer->EndIndex);
+}
+
+_Use_decl_annotations_
+void
+NetvTxQueue::SetNotify(
+ bool NotificationEnabled
+)
+{
+ EnlArmInterrupt(EnlQueueHandle, NotificationEnabled);
+}
diff --git a/network/netadaptercx/netvadapterlibrary/code/txqueue.h b/network/netadaptercx/netvadapterlibrary/code/txqueue.h
new file mode 100644
index 00000000..425a38e9
--- /dev/null
+++ b/network/netadaptercx/netvadapterlibrary/code/txqueue.h
@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved
+
+#include "queue.h"
+
+class NetvTxQueue final
+ : public NetvQueue
+{
+
+public:
+
+ NetvTxQueue(
+ NETPACKETQUEUE Handle,
+ NetvAdapter & Adapter
+ ) noexcept;
+
+ void
+ Destroy(
+ void
+ );
+
+ void
+ Start(
+ void
+ );
+
+ void
+ Stop(
+ void
+ );
+
+ void
+ Advance(
+ void
+ );
+
+ void
+ Cancel(
+ void
+ );
+
+ void
+ SetNotify(
+ bool Enable
+ );
+
+ ENLP_QUEUE * EnlQueueHandle;
+ NET_EXTENSION VirtualAddressExtension;
+ NET_EXTENSION UsoExtension;
+};
+
+WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(NetvTxQueue, NetvTxQueueGetContext);