dwc_eqos - checksum offload

Implement checksum offload.

- Enable the checksum offload engine in hardware if present.
- Disable automatic drop of invalid-checksum packets for the case where
  the Rx offload gets turned off in software.
- Inform NetAdapterCx of the hardware capabilities.
- Add relevant properties to the driver's property sheet. (These are
  handled by NetAdapterCx.)
- Propagate offload information between descriptors and packets.

In testing, I didn't really see any difference in throughput or CPU
usage with this enabled or disabled. Maybe I'm testing wrong.

drivers/net/dwc_eqos/device.cpp

@@ -424,7 +424,8 @@ AdapterCreateTxQueue(
         queueInit,
         context->dma,
         &context->regs->Dma_Ch[0],
-        &context->regs->Mtl_Q[0]);
+        &context->regs->Mtl_Q[0],
+        context->feature0.TxChecksumOffload != 0);
 }
 
 static EVT_NET_ADAPTER_CREATE_RXQUEUE AdapterCreateRxQueue;
@@ -450,7 +451,6 @@ AdapterSetReceiveFilter(
     _In_ NETRECEIVEFILTER receiveFilter)
 {
     // PASSIVE_LEVEL, nonpaged (resume path)
-    TraceEntry(AdapterSetReceiveFilter, LEVEL_INFO);
     auto const context = DeviceGetContext(AdapterGetContext(adapter)->device);
 
     auto const flags = NetReceiveFilterGetPacketFilter(receiveFilter);
@@ -487,11 +487,45 @@ AdapterSetReceiveFilter(
 
     Write32(&context->regs->Mac_Packet_Filter, filter);
 
-    TraceExit(AdapterSetReceiveFilter, LEVEL_INFO,
+    TraceEntryExit(AdapterSetReceiveFilter, LEVEL_INFO,
         TraceLoggingHexInt32(flags),
         TraceLoggingUIntPtr(mcastCount));
 }
 
+static EVT_NET_ADAPTER_OFFLOAD_SET_TX_CHECKSUM AdapterOffloadSetTxChecksum;
+static void
+AdapterOffloadSetTxChecksum(
+    _In_ NETADAPTER adapter,
+    _In_ NETOFFLOAD offload)
+{
+    // PASSIVE_LEVEL, nonpaged (resume path)
+    UNREFERENCED_PARAMETER(adapter);
+    auto const IPv4 = NetOffloadIsTxChecksumIPv4Enabled(offload);
+    auto const Tcp = NetOffloadIsTxChecksumTcpEnabled(offload);
+    auto const Udp = NetOffloadIsTxChecksumUdpEnabled(offload);
+    TraceEntryExit(AdapterOffloadSetTxChecksum, LEVEL_INFO,
+        TraceLoggingBoolean(IPv4),
+        TraceLoggingBoolean(Tcp),
+        TraceLoggingBoolean(Udp));
+}
+
+static EVT_NET_ADAPTER_OFFLOAD_SET_RX_CHECKSUM AdapterOffloadSetRxChecksum;
+static void
+AdapterOffloadSetRxChecksum(
+    _In_ NETADAPTER adapter,
+    _In_ NETOFFLOAD offload)
+{
+    // PASSIVE_LEVEL, nonpaged (resume path)
+    UNREFERENCED_PARAMETER(adapter);
+    auto const IPv4 = NetOffloadIsRxChecksumIPv4Enabled(offload);
+    auto const Tcp = NetOffloadIsRxChecksumTcpEnabled(offload);
+    auto const Udp = NetOffloadIsRxChecksumUdpEnabled(offload);
+    TraceEntryExit(AdapterOffloadSetRxChecksum, LEVEL_INFO,
+        TraceLoggingBoolean(IPv4),
+        TraceLoggingBoolean(Tcp),
+        TraceLoggingBoolean(Udp));
+}
+
 static EVT_WDF_DEVICE_D0_ENTRY DeviceD0Entry;
 static NTSTATUS
 DeviceD0Entry(
@@ -541,7 +575,8 @@ DeviceD0Entry(
 
     MtlRxOperationMode_t rxOperationMode = {};
     rxOperationMode.StoreAndForward = true;
-    rxOperationMode.ForwardErrorPackets = true;
+    rxOperationMode.DisableDropTcpChecksumError = true;
+    rxOperationMode.ForwardErrorPackets = false;
     rxOperationMode.ForwardUndersizedGoodPackets = true;
     rxOperationMode.QueueSize = rxQueueSize / 256u - 1;
     rxOperationMode.HardwareFlowControl = rxQueueSize >= 2048;
@@ -556,6 +591,7 @@ DeviceD0Entry(
     macConfig.PacketBurstEnable = true;
     macConfig.ReceiverEnable = true;
     macConfig.TransmitterEnable = true;
+    macConfig.ChecksumOffloadEnable = context->feature0.TxChecksumOffload | context->feature0.RxChecksumOffload;
     Write32(&context->regs->Mac_Configuration, macConfig);
 
     // Clear and then enable interrupts.
@@ -906,6 +942,27 @@ DevicePrepareHardware(
             NetPacketFilterFlagBroadcast |
             NetPacketFilterFlagPromiscuous;
         NetAdapterSetReceiveFilterCapabilities(context->adapter, &rxFilterCaps);
+
+        if (context->feature0.TxChecksumOffload)
+        {
+            NET_ADAPTER_OFFLOAD_TX_CHECKSUM_CAPABILITIES txChecksumCaps;
+            NET_ADAPTER_OFFLOAD_TX_CHECKSUM_CAPABILITIES_INIT(&txChecksumCaps, {}, AdapterOffloadSetTxChecksum);
+            txChecksumCaps.Layer3Flags =
+                NetAdapterOffloadLayer3FlagIPv4NoOptions |
+                NetAdapterOffloadLayer3FlagIPv4WithOptions |
+                NetAdapterOffloadLayer3FlagIPv6NoExtensions |
+                NetAdapterOffloadLayer3FlagIPv6WithExtensions;
+            txChecksumCaps.Layer4Flags =
+                NetAdapterOffloadLayer4FlagTcpNoOptions |
+                NetAdapterOffloadLayer4FlagTcpWithOptions |
+                NetAdapterOffloadLayer4FlagUdp;
+            NetAdapterOffloadSetTxChecksumCapabilities(context->adapter, &txChecksumCaps);
+        }
+
+        NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES rxChecksumCaps;
+        NET_ADAPTER_OFFLOAD_RX_CHECKSUM_CAPABILITIES_INIT(&rxChecksumCaps,
+            AdapterOffloadSetRxChecksum);
+        NetAdapterOffloadSetRxChecksumCapabilities(context->adapter, &rxChecksumCaps);
     }
 
     // Initialize adapter.
@@ -983,34 +1040,6 @@ DeviceReleaseHardware(
     auto const context = DeviceGetContext(device);
     if (context->regs != nullptr)
     {
-#define CtxStat(x) TraceLoggingUInt32(context->x, #x)
-#define RegStat(x) TraceLoggingUInt32(Read32(&context->regs->x), #x)
-
-        TraceWrite("DeviceReleaseHardware-MacStats", LEVEL_INFO,
-            CtxStat(isrHandled),
-            CtxStat(isrIgnored),
-            CtxStat(dpcLinkState),
-            CtxStat(dpcRx),
-            CtxStat(dpcTx),
-            CtxStat(dpcAbnormalStatus),
-            CtxStat(dpcFatalBusError));
-
-#if 0 // MMC frozen for now
-        TraceWrite("DeviceReleaseHardware-TxStats", LEVEL_INFO,
-            RegStat(TxPacketCountGoodBad),
-            RegStat(TxUnderflowErrorPackets),
-            RegStat(TxCarrierErrorPackets),
-            RegStat(TxPacketCountGood),
-            RegStat(TxPausePackets));
-        TraceWrite("DeviceReleaseHardware-RxStats", LEVEL_INFO,
-            RegStat(RxPacketCountGoodBad),
-            RegStat(RxCrcErrorPackets),
-            RegStat(RxLengthErrorPackets),
-            RegStat(RxPausePackets),
-            RegStat(RxFifoOverflowPackets),
-            RegStat(RxWatchdogErrorPackets));
-#endif
-
         DeviceReset(context->regs, context->permanentMacAddress);
         MmUnmapIoSpace(context->regs, sizeof(*context->regs));
         context->regs = nullptr;

drivers/net/dwc_eqos/driver.cpp

@@ -9,7 +9,6 @@ TODO list:
 - Jumbo frames.
 - Receive queue memory optimization?
 - Configuration in registry (e.g. flow control).
-- Checksum offload.
 - Tx segmentation offload.
 - Rx segmentation offload.
 - Wake-on-LAN.

drivers/net/dwc_eqos/dwc_eqos.inf

@@ -76,13 +76,54 @@ HKR, Ndi\Interfaces, UpperRange, 0, "ndis5"
 HKR, Ndi\Interfaces, LowerRange, 0, "ethernet"
 
 [DWCEQOS_AddReg_Ndi_params]
+
 HKR, Ndi\params\NetworkAddress, ParamDesc, 0, %NetworkAddress%
 HKR, Ndi\params\NetworkAddress, type,      0, "edit"
 HKR, Ndi\params\NetworkAddress, default,   0, ""
 HKR, Ndi\params\NetworkAddress, LimitText, 0, "12"
 HKR, Ndi\params\NetworkAddress, UpperCase, 0, "1"
 HKR, Ndi\params\NetworkAddress, Optional,  0, "1"
 
+HKR, Ndi\params\*IPChecksumOffloadIPv4,          ParamDesc,      0,  %IPChksumOffV4%
+HKR, Ndi\params\*IPChecksumOffloadIPv4,          default,        0,  "3"
+HKR, Ndi\params\*IPChecksumOffloadIPv4,          type,           0,  "enum"
+HKR, Ndi\params\*IPChecksumOffloadIPv4\enum,     "0",            0,  %Disabled%
+HKR, Ndi\params\*IPChecksumOffloadIPv4\enum,     "1",            0,  %TxEnabled%
+HKR, Ndi\params\*IPChecksumOffloadIPv4\enum,     "2",            0,  %RxEnabled%
+HKR, Ndi\params\*IPChecksumOffloadIPv4\enum,     "3",            0,  %RxTxEnabled%
+
+HKR, Ndi\params\*TCPChecksumOffloadIPv4,         ParamDesc,      0,  %TCPChksumOffV4%
+HKR, Ndi\params\*TCPChecksumOffloadIPv4,         default,        0,  "3"
+HKR, Ndi\params\*TCPChecksumOffloadIPv4,         type,           0,  "enum"
+HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum,    "0",            0,  %Disabled%
+HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum,    "1",            0,  %TxEnabled%
+HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum,    "2",            0,  %RxEnabled%
+HKR, Ndi\params\*TCPChecksumOffloadIPv4\enum,    "3",            0,  %RxTxEnabled%
+
+HKR, Ndi\params\*UDPChecksumOffloadIPv4,         ParamDesc,      0,  %UDPChksumOffV4%
+HKR, Ndi\params\*UDPChecksumOffloadIPv4,         default,        0,  "3"
+HKR, Ndi\params\*UDPChecksumOffloadIPv4,         type,           0,  "enum"
+HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum,    "0",            0,  %Disabled%
+HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum,    "1",            0,  %TxEnabled%
+HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum,    "2",            0,  %RxEnabled%
+HKR, Ndi\params\*UDPChecksumOffloadIPv4\enum,    "3",            0,  %RxTxEnabled%
+
+HKR, Ndi\params\*TCPChecksumOffloadIPv6,         ParamDesc,      0,  %TCPChksumOffV6%
+HKR, Ndi\params\*TCPChecksumOffloadIPv6,         default,        0,  "3"
+HKR, Ndi\params\*TCPChecksumOffloadIPv6,         type,           0,  "enum"
+HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum,    "0",            0,  %Disabled%
+HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum,    "1",            0,  %TxEnabled%
+HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum,    "2",            0,  %RxEnabled%
+HKR, Ndi\params\*TCPChecksumOffloadIPv6\enum,    "3",            0,  %RxTxEnabled%
+
+HKR, Ndi\params\*UDPChecksumOffloadIPv6,         ParamDesc,      0,  %UDPChksumOffV6%
+HKR, Ndi\params\*UDPChecksumOffloadIPv6,         default,        0,  "3"
+HKR, Ndi\params\*UDPChecksumOffloadIPv6,         type,           0,  "enum"
+HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum,    "0",            0,  %Disabled%
+HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum,    "1",            0,  %TxEnabled%
+HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum,    "2",            0,  %RxEnabled%
+HKR, Ndi\params\*UDPChecksumOffloadIPv6\enum,    "3",            0,  %RxTxEnabled%
+
 [DWCEQOS_Device.NT.Services]
 AddService = %ServiceName%, 2, DWCEQOS_AddService, DWCEQOS_AddService_EventLog
 
@@ -111,11 +152,20 @@ KmdfService = %ServiceName%, DWCEQOS_KmdfService
 KmdfLibraryVersion = $KMDFVERSION$
 
 [Strings]
-ProviderName = "Open Source"
-NetworkAddress = "Network Address"
-RKCP = "Rockchip"
-DWCEQOS.DeviceDesc = "Synopsys DesignWare Ethernet Quality of Service (GMAC)"
+ProviderName        = "Open Source"
+NetworkAddress      = "Network Address"
+RKCP                = "Rockchip"
+DWCEQOS.DeviceDesc  = "Synopsys DesignWare Ethernet Quality of Service (GMAC)"
 DWCEQOS.ServiceDesc = "DesignWare Ethernet"
+IPChksumOffV4       = "IPv4 Checksum Offload"
+TCPChksumOffV4      = "TCP Checksum Offload (IPv4)"
+UDPChksumOffV4      = "UDP Checksum Offload (IPv4)"
+TCPChksumOffV6      = "TCP Checksum Offload (IPv6)"
+UDPChksumOffV6      = "UDP Checksum Offload (IPv6)"
+Disabled            = "Disabled"
+TxEnabled           = "Tx Enabled"
+RxEnabled           = "Rx Enabled"
+RxTxEnabled         = "Rx & Tx Enabled"
 
 ; Not localized
 ServiceName = "dwc_eqos"

drivers/net/dwc_eqos/precomp.h

@@ -10,6 +10,7 @@
 #include <netadaptercx.h>
 #include <net/logicaladdress.h>
 #include <net/virtualaddress.h>
+#include <net/checksum.h>
 #include <initguid.h>
 #include <TraceLoggingProvider.h>
 #include <winmeta.h>

drivers/net/dwc_eqos/registers.h

@@ -1735,7 +1735,6 @@ constexpr unsigned DESCRIPTOR_ALIGN = 64u;
 #define RX_DESCRIPTOR_SIZE 16
 #endif
 
-
 enum TxChecksumInsertion : UINT16
 {
     TxChecksumInsertionDisabled = 0,
@@ -1937,6 +1936,18 @@ struct RxDescriptorRead
 };
 static_assert(sizeof(RxDescriptorRead) == RX_DESCRIPTOR_SIZE);
 
+enum RxPayloadType : UINT8
+{
+    RxPayloadTypeUnknown = 0,
+    RxPayloadTypeUdp,
+    RxPayloadTypeTcp,
+    RxPayloadTypeIcmp,
+    RxPayloadTypeIgmp,
+    RxPayloadTypeAvUntaggedControl,
+    RxPayloadTypeAvTaggedData,
+    RxPayloadTypeAvTaggedControl,
+};
+
 struct RxDescriptorWrite
 {
     // RDES0
@@ -1946,7 +1957,7 @@ struct RxDescriptorWrite
 
     // RDES1
 
-    UINT8 PayloadType : 3; // PT
+    RxPayloadType PayloadType : 3; // PT
     UINT8 IPHeaderError : 1; // IPHE
     UINT8 IPv4HeaderPresent : 1; // IPV4
     UINT8 IPv6HeaderPresent : 1; // IPV6

drivers/net/dwc_eqos/rxqueue.cpp

@@ -17,6 +17,7 @@ struct RxQueueContext
     WDFCOMMONBUFFER descBuffer;
     RxDescriptor* descVirtual;
     PHYSICAL_ADDRESS descPhysical;
+    NET_EXTENSION packetChecksum;
     NET_EXTENSION fragmentLogical;
     UINT32 descCount;   // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount.
     bool running;
@@ -167,6 +168,39 @@ RxQueueAdvance(_In_ NETPACKETQUEUE queue)
         {
             NT_ASSERT(descWrite.PacketLength >= 4); // PacketLength includes CRC
             frag->ValidLength = descWrite.PacketLength - 4;
+
+            // If checksum offload is disabled by hardware then no IP headers will be
+            // detected. If checksum offload is disabled by software then NetAdapterCx
+            // will ignore our evaluation.
+            if (descWrite.IPv4HeaderPresent | descWrite.IPv6HeaderPresent)
+            {
+                auto const checksum = NetExtensionGetPacketChecksum(&context->packetChecksum, pktIndex);
+                checksum->Layer2 = NetPacketRxChecksumEvaluationValid;
+                pkt->Layout.Layer2Type = NetPacketLayer2TypeEthernet;
+
+                checksum->Layer3 = descWrite.IPHeaderError
+                    ? NetPacketRxChecksumEvaluationInvalid
+                    : NetPacketRxChecksumEvaluationValid;
+                pkt->Layout.Layer3Type = descWrite.IPv4HeaderPresent
+                    ? NetPacketLayer3TypeIPv4UnspecifiedOptions
+                    : NetPacketLayer3TypeIPv6UnspecifiedExtensions;
+
+                checksum->Layer4 = descWrite.IPPayloadError
+                    ? NetPacketRxChecksumEvaluationInvalid
+                    : NetPacketRxChecksumEvaluationValid;
+                switch (descWrite.PayloadType)
+                {
+                case RxPayloadTypeUdp:
+                    pkt->Layout.Layer4Type = NetPacketLayer4TypeUdp;
+                    break;
+                case RxPayloadTypeTcp:
+                    pkt->Layout.Layer4Type = NetPacketLayer4TypeTcp;
+                    break;
+                default:
+                    pkt->Layout.Layer4Type = NetPacketLayer4TypeUnspecified;
+                    break;
+                }
+            }
         }
 
         pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex);
@@ -377,6 +411,13 @@ RxQueueCreate(
             TraceLoggingPointer(context->descVirtual, "virtual"));
 
         NET_EXTENSION_QUERY query;
+
+        NET_EXTENSION_QUERY_INIT(&query,
+            NET_PACKET_EXTENSION_CHECKSUM_NAME,
+            NET_PACKET_EXTENSION_CHECKSUM_VERSION_1,
+            NetExtensionTypePacket);
+        NetRxQueueGetExtension(queue, &query, &context->packetChecksum);
+
         NET_EXTENSION_QUERY_INIT(&query,
             NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_NAME,
             NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_VERSION_1,