From 7dfb92c9e6b8f968a93ebde7b994eded77782adc Mon Sep 17 00:00:00 2001 From: "Doug Cook (WINDOWS)" Date: Thu, 23 Nov 2023 18:38:08 -0800 Subject: [PATCH] dwc_eqos - driver for RK3588 GMAC Basic NetAdapterCx 2.2 driver for the Ethernet MAC in the RK3588. Works for 1Gbps. Potential TODO list: - Support for 10/100 Mbps modes (requires ACPI support). - Use ACPI to get DMA configuration. - Jumbo frames. - Receive queue memory optimization. - Configuration options (e.g. enable/disable flow control, etc.). - Checksum offload. - Tx segmentation offload. - Rx segmentation offload. - Wake-on-LAN. - ARP offload. - Multi-queue support. --- build/RockchipDrivers.sln | 8 + drivers/net/dwc_eqos/LICENSE | 21 + drivers/net/dwc_eqos/_precomp.cpp | 1 + drivers/net/dwc_eqos/device.cpp | 1100 +++++++++ drivers/net/dwc_eqos/device.h | 15 + drivers/net/dwc_eqos/driver.cpp | 74 + drivers/net/dwc_eqos/dwc_eqos.inf | 121 + drivers/net/dwc_eqos/dwc_eqos.vcxproj | 121 + drivers/net/dwc_eqos/dwc_eqos.vcxproj.filters | 69 + drivers/net/dwc_eqos/precomp.h | 15 + drivers/net/dwc_eqos/queue_common.cpp | 37 + drivers/net/dwc_eqos/queue_common.h | 20 + drivers/net/dwc_eqos/registers.h | 2024 +++++++++++++++++ drivers/net/dwc_eqos/rxqueue.cpp | 379 +++ drivers/net/dwc_eqos/rxqueue.h | 14 + drivers/net/dwc_eqos/trace.h | 48 + drivers/net/dwc_eqos/txqueue.cpp | 480 ++++ drivers/net/dwc_eqos/txqueue.h | 15 + 18 files changed, 4562 insertions(+) create mode 100644 drivers/net/dwc_eqos/LICENSE create mode 100644 drivers/net/dwc_eqos/_precomp.cpp create mode 100644 drivers/net/dwc_eqos/device.cpp create mode 100644 drivers/net/dwc_eqos/device.h create mode 100644 drivers/net/dwc_eqos/driver.cpp create mode 100644 drivers/net/dwc_eqos/dwc_eqos.inf create mode 100644 drivers/net/dwc_eqos/dwc_eqos.vcxproj create mode 100644 drivers/net/dwc_eqos/dwc_eqos.vcxproj.filters create mode 100644 drivers/net/dwc_eqos/precomp.h create mode 100644 drivers/net/dwc_eqos/queue_common.cpp create mode 100644 drivers/net/dwc_eqos/queue_common.h create mode 100644 drivers/net/dwc_eqos/registers.h create mode 100644 drivers/net/dwc_eqos/rxqueue.cpp create mode 100644 drivers/net/dwc_eqos/rxqueue.h create mode 100644 drivers/net/dwc_eqos/trace.h create mode 100644 drivers/net/dwc_eqos/txqueue.cpp create mode 100644 drivers/net/dwc_eqos/txqueue.h diff --git a/build/RockchipDrivers.sln b/build/RockchipDrivers.sln index e4b1e67..20c7c23 100644 --- a/build/RockchipDrivers.sln +++ b/build/RockchipDrivers.sln @@ -27,6 +27,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "csaudio-Main", "..\drivers\ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "csaudio-Utilities", "..\drivers\audio\csaudiork3x\Source\Utilities\Utilities.vcxproj", "{33E61864-6F2C-4F9F-BE70-8F8985A4F283}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dwc_eqos", "..\drivers\net\dwc_eqos\dwc_eqos.vcxproj", "{CAD465B6-97D7-402D-85F8-50DE140A6D9D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64 = Debug|ARM64 @@ -91,6 +93,12 @@ Global {33E61864-6F2C-4F9F-BE70-8F8985A4F283}.Release|ARM64.ActiveCfg = Release|ARM64 {33E61864-6F2C-4F9F-BE70-8F8985A4F283}.Release|ARM64.Build.0 = Release|ARM64 {33E61864-6F2C-4F9F-BE70-8F8985A4F283}.Release|ARM64.Deploy.0 = Release|ARM64 + {CAD465B6-97D7-402D-85F8-50DE140A6D9D}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {CAD465B6-97D7-402D-85F8-50DE140A6D9D}.Debug|ARM64.Build.0 = Debug|ARM64 + {CAD465B6-97D7-402D-85F8-50DE140A6D9D}.Debug|ARM64.Deploy.0 = Debug|ARM64 + {CAD465B6-97D7-402D-85F8-50DE140A6D9D}.Release|ARM64.ActiveCfg = Release|ARM64 + {CAD465B6-97D7-402D-85F8-50DE140A6D9D}.Release|ARM64.Build.0 = Release|ARM64 + {CAD465B6-97D7-402D-85F8-50DE140A6D9D}.Release|ARM64.Deploy.0 = Release|ARM64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/drivers/net/dwc_eqos/LICENSE b/drivers/net/dwc_eqos/LICENSE new file mode 100644 index 0000000..6b124e7 --- /dev/null +++ b/drivers/net/dwc_eqos/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Doug Cook. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/drivers/net/dwc_eqos/_precomp.cpp b/drivers/net/dwc_eqos/_precomp.cpp new file mode 100644 index 0000000..ceeb0d6 --- /dev/null +++ b/drivers/net/dwc_eqos/_precomp.cpp @@ -0,0 +1 @@ +#include "precomp.h" diff --git a/drivers/net/dwc_eqos/device.cpp b/drivers/net/dwc_eqos/device.cpp new file mode 100644 index 0000000..44ca739 --- /dev/null +++ b/drivers/net/dwc_eqos/device.cpp @@ -0,0 +1,1100 @@ +#include "precomp.h" +#include "device.h" +#include "rxqueue.h" +#include "txqueue.h" +#include "queue_common.h" +#include "registers.h" +#include "trace.h" + +#include + +/* +Lifecycle: + +PrepareHardware ReleaseHardware +D0Entry D0Exit +(InterruptEnable) (InterruptDisable) +CreateRxQueue DestroyCallback +CreateTxQueue DestroyCallback +(PacketQueueStart) (PacketQueueCancel, PacketQueueStop) +(DisarmWake) (ArmWake) +*/ + +static auto constexpr DefaultAxiMaxWriteOutstanding = 4u; +static auto constexpr DefaultAxiMaxReadOutstanding = 8u; +static auto constexpr DefaultCsrRate = 125'000'000u; +static auto constexpr BusBytes = 8u; +static auto constexpr LinkStatusBit = 0x80000000u; + +enum InterruptsWanted : char +{ + InterruptsNone = 0, + InterruptsState = 1 << 0, // mac.LinkStatus, ch0.AbnormalInterruptSummary, ch0.FatalBusError + InterruptsRx = 1 << 1, // ch0.Rx + InterruptsTx = 1 << 2, // ch0.Tx + InterruptsAll = -1, +}; + +struct DeviceContext +{ + // Const after initialization. + + MacRegisters* regs; + NETADAPTER adapter; + WDFSPINLOCK lock; + WDFINTERRUPT interrupt; + WDFDMAENABLER dma; + MacHwFeature0_t feature0; + MacHwFeature1_t feature1; + MacHwFeature2_t feature2; + MacHwFeature3_t feature3; + UINT8 permanentMacAddress[ETHERNET_LENGTH_OF_ADDRESS]; + UINT8 currentMacAddress[ETHERNET_LENGTH_OF_ADDRESS]; + + // Mutable. + + LONG interruptStatus; // = ChannelStatus_t, plus top bit is LinkStatus. Interlocked update. + + InterruptsWanted interruptsWanted; // Guarded by lock. + NETPACKETQUEUE rxQueue; // Guarded by lock. + NETPACKETQUEUE txQueue; // Guarded by lock. + + // Diagnostics/statistics. + + UINT32 isrHandled; // Updated only in ISR. + UINT32 isrIgnored; // Updated only in ISR. + UINT32 dpcLinkState; // Updated only in DPC. + UINT32 dpcRx; // Updated only in DPC. + UINT32 dpcTx; // Updated only in DPC. + UINT32 dpcAbnormalStatus; // Updated only in DPC. + UINT32 dpcFatalBusError; // Updated only in DPC. +}; +WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(DeviceContext, DeviceGetContext) + +struct AdapterContext +{ + WDFDEVICE device; +}; +WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(AdapterContext, AdapterGetContext) + +static void +SetOneMacAddress(_Inout_ MacRegisters* regs, unsigned index, _In_reads_(6) UINT8 const* addr, bool enable) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + MacAddressLow_t regLo = {}; + regLo.Addr0 = addr[0]; + regLo.Addr1 = addr[1]; + regLo.Addr2 = addr[2]; + regLo.Addr3 = addr[3]; + + MacAddressHigh_t regHi = {}; + regHi.Addr4 = addr[4]; + regHi.Addr5 = addr[5]; + regHi.AddressEnable = enable; + + Write32(®s->MacAddress[index].High, regHi); + Write32(®s->MacAddress[index].Low, regLo); + + TraceEntryExit(SetOneMacAddress, LEVEL_VERBOSE, + TraceLoggingUInt32(index), + TraceLoggingHexInt32(regHi.Value32, "MacHi"), + TraceLoggingHexInt32(regLo.Value32, "MacLo")); +} + +// Perform a software reset, then set mac address 0 to the specified value. +// Returns either STATUS_SUCCESS or STATUS_TIMEOUT. +_IRQL_requires_(PASSIVE_LEVEL) +__declspec(code_seg("PAGE")) +static NTSTATUS +DeviceReset(_Inout_ MacRegisters* regs, _In_reads_(6) UINT8 const* mac0) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + + Write32(®s->DmaMode, 1); // Software reset. + + unsigned retry; + for (retry = 1000u; retry != 0; retry -= 1) + { + KeStallExecutionProcessor(20); + auto const dmaMode = Read32(®s->DmaMode); + if (0 == (dmaMode & 1)) + { + SetOneMacAddress(regs, 0, mac0, true); + TraceEntryExit(DeviceReset, LEVEL_INFO, + TraceLoggingUInt32(retry)); + return STATUS_SUCCESS; + } + } + + TraceWrite("DeviceReset-timeout", LEVEL_ERROR); + return STATUS_TIMEOUT; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static void +UpdateLinkState(_In_ DeviceContext const* context) +{ + // DISPATCH_LEVEL + auto const controlStatus = Read32(&context->regs->MacPhyIfControlStatus); // Clears LinkStatus interrupt. + auto const oldConfig = Read32(&context->regs->MacConfiguration); + auto newConfig = oldConfig; + newConfig.FullDuplex = controlStatus.FullDuplex; + + UINT32 speed; + switch (controlStatus.Speed) + { + case PhyIfSpeed_2_5M: + speed = 10'000'000u; + newConfig.PortSelectSpeed = PortSelectSpeed_10M; + break; + case PhyIfSpeed_25M: + speed = 100'000'000u; + newConfig.PortSelectSpeed = PortSelectSpeed_100M; + break; + case PhyIfSpeed_125M: + speed = 1'000'000'000u; + newConfig.PortSelectSpeed = PortSelectSpeed_1000M; + break; + default: + speed = 0; + break; + } + + // TODO: I think this is where we want to call ACPI to change phy clock speed. + + if (oldConfig.Value32 != newConfig.Value32) + { + Write32(&context->regs->MacConfiguration, newConfig); + } + + NET_ADAPTER_LINK_STATE linkState; + NET_ADAPTER_LINK_STATE_INIT( + &linkState, + speed, + controlStatus.LinkUp ? MediaConnectStateConnected : MediaConnectStateDisconnected, + controlStatus.FullDuplex ? MediaDuplexStateFull : MediaDuplexStateHalf, + NetAdapterPauseFunctionTypeUnsupported, // TODO: Pause functions? + NetAdapterAutoNegotiationFlagXmitLinkSpeedAutoNegotiated | + NetAdapterAutoNegotiationFlagRcvLinkSpeedautoNegotiated | + NetAdapterAutoNegotiationFlagDuplexAutoNegotiated); + NetAdapterSetLinkState(context->adapter, &linkState); + + TraceEntryExit(UpdateLinkState, LEVEL_INFO, + TraceLoggingHexInt32(controlStatus.Value32, "PhyIfControlStatus"), + TraceLoggingHexInt32(oldConfig.Value32, "OldMacConfig"), + TraceLoggingHexInt32(newConfig.Value32, "NewMacConfig")); +} + +// Cleared by reading MacPhyIfControlStatus. +_IRQL_requires_max_(HIGH_LEVEL) +static MacInterruptEnable_t +MakeMacInterruptEnable(char interruptsWanted) +{ + // HIGH_LEVEL + MacInterruptEnable_t interruptEnable = {}; + interruptEnable.LinkStatus = 0 != (interruptsWanted & InterruptsState); + return interruptEnable; +} + +// Cleared by writing Channel.Status. +_IRQL_requires_max_(HIGH_LEVEL) +static ChannelInterruptEnable_t +MakeChannelInterruptEnable(char interruptsWanted) +{ + // HIGH_LEVEL + ChannelInterruptEnable_t interruptEnable = {}; + interruptEnable.Rx = 0 != (interruptsWanted & InterruptsRx); + interruptEnable.Tx = 0 != (interruptsWanted & InterruptsTx); + interruptEnable.NormalInterruptSummary = 1; + interruptEnable.FatalBusError = 0 != (interruptsWanted & InterruptsState); + interruptEnable.AbnormalInterruptSummary = 0 != (interruptsWanted & InterruptsState); + return interruptEnable; +} + +_IRQL_requires_(DISPATCH_LEVEL) +static void +DeviceInterruptEnable_Locked(_Inout_ DeviceContext* context, InterruptsWanted bitsToEnable) +{ + // DISPATCH_LEVEL + auto const oldWanted = context->interruptsWanted; + auto const newWanted = static_cast(oldWanted | bitsToEnable); + if (oldWanted != newWanted) + { + context->interruptsWanted = newWanted; + Write32(&context->regs->MacInterruptEnable, MakeMacInterruptEnable(newWanted)); + Write32(&context->regs->DmaCh[0].InterruptEnable, MakeChannelInterruptEnable(newWanted)); + TraceEntryExit(DeviceInterruptEnable, LEVEL_VERBOSE, + TraceLoggingHexInt32(oldWanted, "old"), + TraceLoggingHexInt32(newWanted, "new")); + } +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static void +DeviceInterruptEnable(_Inout_ DeviceContext* context, InterruptsWanted bitsToEnable) +{ + // DISPATCH_LEVEL + WdfSpinLockAcquire(context->lock); + DeviceInterruptEnable_Locked(context, bitsToEnable); + WdfSpinLockRelease(context->lock); +} + +_IRQL_requires_(DISPATCH_LEVEL) +static void +DeviceInterruptDisable_Locked(_Inout_ DeviceContext* context, InterruptsWanted bitsToDisable) +{ + // DISPATCH_LEVEL + auto const oldWanted = context->interruptsWanted; + auto const newWanted = static_cast(oldWanted & ~bitsToDisable); + if (oldWanted != newWanted) + { + context->interruptsWanted = newWanted; + Write32(&context->regs->MacInterruptEnable, MakeMacInterruptEnable(newWanted)); + Write32(&context->regs->DmaCh[0].InterruptEnable, MakeChannelInterruptEnable(newWanted)); + TraceWrite("DeviceInterruptDisable", LEVEL_VERBOSE, + TraceLoggingHexInt32(oldWanted, "old"), + TraceLoggingHexInt32(newWanted, "new")); + } +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +static void +DeviceInterruptDisable(_Inout_ DeviceContext* context, InterruptsWanted bitsToEnable) +{ + // DISPATCH_LEVEL + WdfSpinLockAcquire(context->lock); + DeviceInterruptDisable_Locked(context, bitsToEnable); + WdfSpinLockRelease(context->lock); +} + +static EVT_WDF_INTERRUPT_ISR DeviceInterruptIsr; +static BOOLEAN +DeviceInterruptIsr( + _In_ WDFINTERRUPT interrupt, + _In_ ULONG messageId) +{ + // HIGH_LEVEL + UNREFERENCED_PARAMETER(messageId); + auto const context = DeviceGetContext(WdfInterruptGetDevice(interrupt)); + + auto const mac = Read32(&context->regs->MacInterruptStatus); + auto const channel0 = Read32(&context->regs->DmaCh[0].Status); + if (mac.LinkStatus || channel0.Value32 != 0) + { + if (mac.LinkStatus) + { + Read32(&context->regs->MacPhyIfControlStatus); // Clears LinkStatus. + } + + if (channel0.Value32 != 0) + { + Write32(&context->regs->DmaCh[0].Status, channel0); // Clears DmaCh0.Status. + } + + InterlockedOrNoFence( + &context->interruptStatus, + channel0.Value32 | (mac.LinkStatus ? LinkStatusBit : 0u)); + WdfInterruptQueueDpcForIsr(interrupt); + + context->isrHandled += 1; + return true; + } + + context->isrIgnored += 1; + return false; +} + +static EVT_WDF_INTERRUPT_DPC DeviceInterruptDpc; +static void +DeviceInterruptDpc( + _In_ WDFINTERRUPT interrupt, + _In_ WDFOBJECT associatedObject) +{ + // DISPATCH_LEVEL + UNREFERENCED_PARAMETER(interrupt); + auto const context = DeviceGetContext(associatedObject); + NT_ASSERT(context->interrupt == interrupt); + + for (;;) + { + ChannelStatus_t const status{ (ULONG)InterlockedExchangeNoFence(&context->interruptStatus, 0) }; + if (status.Value32 == 0) + { + break; + } + + if (status.AbnormalInterruptSummary || status.FatalBusError) + { + // TODO - error recovery? + context->dpcAbnormalStatus += status.AbnormalInterruptSummary; + context->dpcFatalBusError += status.FatalBusError; + TraceWrite("DeviceInterruptDpc-ERROR", LEVEL_ERROR, + TraceLoggingHexInt32(status.Value32, "status")); + } + else + { + TraceWrite("DeviceInterruptDpc", LEVEL_VERBOSE, + TraceLoggingHexInt32(status.Value32, "status")); + } + + if (status.Value32 & LinkStatusBit) + { + context->dpcLinkState += 1; + UpdateLinkState(context); + } + + auto const interruptsRxTx = static_cast( + (status.Rx ? InterruptsRx : InterruptsNone) | + (status.Tx ? InterruptsTx : InterruptsNone)); + if (interruptsRxTx != 0) + { + WdfSpinLockAcquire(context->lock); + + DeviceInterruptDisable_Locked(context, interruptsRxTx); + + if (status.Rx && context->rxQueue) + { + context->dpcRx += 1; + NetRxQueueNotifyMoreReceivedPacketsAvailable(context->rxQueue); + context->rxQueue = nullptr; + } + + if (status.Tx && context->txQueue) + { + context->dpcTx += 1; + NetTxQueueNotifyMoreCompletedPacketsAvailable(context->txQueue); + context->txQueue = nullptr; + } + + WdfSpinLockRelease(context->lock); + } + } +} + +static EVT_NET_ADAPTER_CREATE_TXQUEUE AdapterCreateTxQueue; +static NTSTATUS +AdapterCreateTxQueue( + _In_ NETADAPTER adapter, + _Inout_ NETTXQUEUE_INIT* queueInit) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = DeviceGetContext(AdapterGetContext(adapter)->device); + NT_ASSERT(context->txQueue == nullptr); + return TxQueueCreate( + adapter, + queueInit, + context->dma, + &context->regs->DmaCh[0], + &context->regs->MtlQ[0]); +} + +static EVT_NET_ADAPTER_CREATE_RXQUEUE AdapterCreateRxQueue; +static NTSTATUS +AdapterCreateRxQueue( + _In_ NETADAPTER adapter, + _Inout_ NETRXQUEUE_INIT* queueInit) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = DeviceGetContext(AdapterGetContext(adapter)->device); + NT_ASSERT(context->rxQueue == nullptr); + return RxQueueCreate( + adapter, + queueInit, + context->dma, + &context->regs->DmaCh[0]); +} + +static EVT_NET_ADAPTER_SET_RECEIVE_FILTER AdapterSetReceiveFilter; +static void +AdapterSetReceiveFilter( + _In_ NETADAPTER adapter, + _In_ NETRECEIVEFILTER receiveFilter) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + TraceEntry(AdapterSetReceiveFilter, LEVEL_INFO); + auto const context = DeviceGetContext(AdapterGetContext(adapter)->device); + + auto const flags = NetReceiveFilterGetPacketFilter(receiveFilter); + auto const mcastCount = (flags & NetPacketFilterFlagMulticast) + ? NetReceiveFilterGetMulticastAddressCount(receiveFilter) + : 0; + auto const mcast = mcastCount > 0 + ? NetReceiveFilterGetMulticastAddressList(receiveFilter) + : nullptr; + + MacPacketFilter_t filter = {}; + if (flags & NetPacketFilterFlagPromiscuous) + { + filter.PromiscuousMode = true; + } + else + { + filter.PassAllMulticast = 0 != (flags & NetPacketFilterFlagAllMulticast); + filter.DisableBroadcast = 0 == (flags & NetPacketFilterFlagBroadcast); + + SetOneMacAddress(context->regs, 0, context->currentMacAddress, + 0 != (flags & NetPacketFilterFlagDirected)); // Address[0] can't really be disabled... + + // Could also use hash-based filtering for additional mcast support, but this seems okay. + auto const macAddrCount = context->feature0.MacAddrCount; + for (unsigned i = 1; i < macAddrCount; i += 1) + { + static constexpr UINT8 zero[ETHERNET_LENGTH_OF_ADDRESS] = {}; + bool const enable = mcastCount > i - 1 && mcast[i - 1].Length >= ETHERNET_LENGTH_OF_ADDRESS; + auto const addr = enable ? mcast[i - 1].Address : zero; + SetOneMacAddress(context->regs, i, addr, enable); + } + } + + Write32(&context->regs->MacPacketFilter, filter); + + TraceExit(AdapterSetReceiveFilter, LEVEL_INFO, + TraceLoggingHexInt32(flags), + TraceLoggingUIntPtr(mcastCount)); +} + +static EVT_WDF_DEVICE_D0_ENTRY DeviceD0Entry; +static NTSTATUS +DeviceD0Entry( + _In_ WDFDEVICE device, + _In_ WDF_POWER_DEVICE_STATE previousState) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + NTSTATUS status = STATUS_SUCCESS; + auto const context = DeviceGetContext(device); + + // TX configuration. + + MacTxFlowCtrl_t txFlowCtrl = {}; + txFlowCtrl.TransmitFlowControlEnable = true; + txFlowCtrl.PauseTime = 0xFFFF; + Write32(&context->regs->MacTxFlowCtrl, txFlowCtrl); // TxFlow control, pause time. + + MtlTxOperationMode_t txOperationMode = {}; + txOperationMode.StoreAndForward = true; + txOperationMode.QueueEnable = MtlTxQueueEnable_Enabled; + txOperationMode.QueueSize = (128u << context->feature1.TxFifoSize) / 256u - 1; // Use 100% of FIFO. (TODO: Not sure about the -1.) + Write32(&context->regs->MtlQ[0].TxOperationMode, txOperationMode); + + // RX configuration. + + Write32(&context->regs->MacRxFlowCtrl, 0x3); // Rx flow control, pause packet detect. + Write32(&context->regs->MacRxCtrl0, 0x2); // RxQ0 enabled for DCB/generic. + + MtlRxOperationMode_t rxOperationMode = {}; + rxOperationMode.StoreAndForward = true; + rxOperationMode.ForwardErrorPackets = true; + rxOperationMode.ForwardUndersizedGoodPackets = true; + rxOperationMode.QueueSize = (128u << context->feature1.RxFifoSize) / 256u - 1; // Use 100% of FIFO. (TODO: Not sure about the -1.) + rxOperationMode.HardwareFlowControl = true; + rxOperationMode.FlowControlActivate = 2; // Full - 2KB + rxOperationMode.FlowControlDeactivate = 10; // Full - 6KB + Write32(&context->regs->MtlQ[0].RxOperationMode, rxOperationMode); + + // MAC configuration. + + MacConfiguration_t macConfig = {}; + macConfig.DisableCarrierSenseDuringTransmit = true; + macConfig.PacketBurstEnable = true; + macConfig.ReceiverEnable = true; + macConfig.TransmitterEnable = true; + Write32(&context->regs->MacConfiguration, macConfig); + + // Clear and then enable interrupts. + UpdateLinkState(context); + (void)Read32(&context->regs->DmaCh[0].Status); + DeviceInterruptEnable(context, InterruptsState); + + TraceEntryExitWithStatus(DeviceD0Entry, LEVEL_INFO, status, + TraceLoggingUInt32(previousState)); + return status; +} + +static EVT_WDF_DEVICE_D0_EXIT DeviceD0Exit; +__declspec(code_seg("PAGE")) +static NTSTATUS +DeviceD0Exit( + _In_ WDFDEVICE device, + _In_ WDF_POWER_DEVICE_STATE targetState) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + NTSTATUS status = STATUS_SUCCESS; + auto const context = DeviceGetContext(device); + + DeviceInterruptDisable(context, InterruptsAll); + + NT_ASSERT(context->txQueue == nullptr); + NT_ASSERT(context->rxQueue == nullptr); + + auto macConfig = Read32(&context->regs->MacConfiguration); + macConfig.ReceiverEnable = false; + macConfig.TransmitterEnable = false; + Write32(&context->regs->MacConfiguration, macConfig); + + TraceEntryExitWithStatus(DeviceD0Exit, LEVEL_INFO, status, + TraceLoggingUInt32(targetState)); + return status; +} + +static EVT_WDF_DEVICE_PREPARE_HARDWARE DevicePrepareHardware; +__declspec(code_seg("PAGE")) +static NTSTATUS +DevicePrepareHardware( + _In_ WDFDEVICE device, + _In_ WDFCMRESLIST resourcesRaw, + _In_ WDFCMRESLIST resourcesTranslated) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + UNREFERENCED_PARAMETER(resourcesRaw); + + NTSTATUS status; + PHYSICAL_ADDRESS maxPhysicalAddress; + auto const context = DeviceGetContext(device); + bool configHasMacAddress = false; + + // Read configuration + + { + NETCONFIGURATION configuration; + status = NetAdapterOpenConfiguration(context->adapter, WDF_NO_OBJECT_ATTRIBUTES, &configuration); + if (!NT_SUCCESS(status)) + { + TraceWrite("NetAdapterOpenConfiguration-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + + NET_ADAPTER_LINK_LAYER_ADDRESS configAddress; + status = NetConfigurationQueryLinkLayerAddress(configuration, &configAddress); + if (!NT_SUCCESS(status)) + { + TraceWrite("QueryLinkLayerAddress-not-found", LEVEL_VERBOSE, + TraceLoggingNTStatus(status)); + } + else if (configAddress.Length != ETHERNET_LENGTH_OF_ADDRESS) + { + TraceWrite("QueryLinkLayerAddress-bad-length", LEVEL_WARNING, + TraceLoggingHexInt16(configAddress.Length, "Length")); + } + else if ( + ETH_IS_MULTICAST(configAddress.Address) || + ETH_IS_BROADCAST(configAddress.Address)) + { + TraceWrite("QueryLinkLayerAddress-bad-address", LEVEL_WARNING, + TraceLoggingBinary(configAddress.Address, ETHERNET_LENGTH_OF_ADDRESS, "address")); + } + else + { + TraceWrite("QueryLinkLayerAddress-found", LEVEL_INFO, + TraceLoggingBinary(configAddress.Address, ETHERNET_LENGTH_OF_ADDRESS, "address")); + memcpy(context->currentMacAddress, configAddress.Address, sizeof(context->currentMacAddress)); + configHasMacAddress = true; + } + } + + // Configure resources + + { + NT_ASSERT(context->regs == nullptr); + NT_ASSERT(context->interrupt == nullptr); + + unsigned interruptsFound = 0; + ULONG const resourcesCount = WdfCmResourceListGetCount(resourcesTranslated); + for (ULONG i = 0; i != resourcesCount; i += 1) + { + auto desc = WdfCmResourceListGetDescriptor(resourcesTranslated, i); + switch (desc->Type) + { + case CmResourceTypeMemory: + if (context->regs != nullptr) + { + TraceWrite("DevicePrepareHardware-memory-unexpected", LEVEL_WARNING, + TraceLoggingHexInt64(desc->u.Memory.Start.QuadPart, "start")); + } + else if (desc->u.Memory.Length < sizeof(*context->regs)) + { + TraceWrite("DevicePrepareHardware-memory-small", LEVEL_WARNING, + TraceLoggingHexInt64(desc->u.Memory.Start.QuadPart, "start"), + TraceLoggingHexInt32(desc->u.Memory.Length, "length")); + } + else + { + TraceWrite("DevicePrepareHardware-memory", LEVEL_VERBOSE, + TraceLoggingHexInt64(desc->u.Memory.Start.QuadPart, "start"), + TraceLoggingHexInt32(desc->u.Memory.Length, "length")); + + context->regs = static_cast( + MmMapIoSpaceEx(desc->u.Memory.Start, sizeof(*context->regs), PAGE_READWRITE | PAGE_NOCACHE)); + if (context->regs == nullptr) + { + TraceWrite("MmMapIoSpaceEx-failed", LEVEL_ERROR); + status = STATUS_INSUFFICIENT_RESOURCES; + goto Done; + } + } + break; + + case CmResourceTypeInterrupt: + switch (interruptsFound++) + { + case 0: + TraceWrite("DevicePrepareHardware-interrupt-sbd", LEVEL_VERBOSE, + TraceLoggingHexInt32(desc->u.Interrupt.Vector, "vector")); + + WDF_INTERRUPT_CONFIG config; + WDF_INTERRUPT_CONFIG_INIT(&config, DeviceInterruptIsr, DeviceInterruptDpc); + config.InterruptRaw = WdfCmResourceListGetDescriptor(resourcesRaw, i); + config.InterruptTranslated = desc; + + status = WdfInterruptCreate(device, &config, WDF_NO_OBJECT_ATTRIBUTES, &context->interrupt); + if (!NT_SUCCESS(status)) + { + TraceWrite("WdfInterruptCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + break; + case 1: + TraceWrite("DevicePrepareHardware-interrupt-pmt", LEVEL_VERBOSE, + TraceLoggingHexInt32(desc->u.Interrupt.Vector, "vector")); + break; + default: + TraceWrite("DevicePrepareHardware-interrupt-unexpected", LEVEL_WARNING, + TraceLoggingHexInt32(desc->u.Interrupt.Vector, "vector")); + break; + } + break; + + default: + TraceWrite("DevicePrepareHardware-resource-unexpected", LEVEL_WARNING, + TraceLoggingUInt8(desc->Type, "type")); + break; + } + } + } + + if (context->regs == nullptr) + { + TraceWrite("DevicePrepareHardware-no-memory", LEVEL_ERROR); + status = STATUS_DEVICE_CONFIGURATION_ERROR; + goto Done; + } + + if (context->interrupt == nullptr) + { + TraceWrite("DevicePrepareHardware-no-interrupt", LEVEL_ERROR); + status = STATUS_DEVICE_CONFIGURATION_ERROR; + goto Done; + } + + // Set up MAC address + + { + auto const mac0hi = Read32(&context->regs->MacAddress[0].High); + auto const mac0lo = Read32(&context->regs->MacAddress[0].Low); + context->permanentMacAddress[0] = mac0lo.Addr0; + context->permanentMacAddress[1] = mac0lo.Addr1; + context->permanentMacAddress[2] = mac0lo.Addr2; + context->permanentMacAddress[3] = mac0lo.Addr3; + context->permanentMacAddress[4] = mac0hi.Addr4; + context->permanentMacAddress[5] = mac0hi.Addr5; + + if (!configHasMacAddress) + { + memcpy(context->currentMacAddress, context->permanentMacAddress, sizeof(context->currentMacAddress)); + if ((mac0lo.Value32 == 0xFFFFFFFF && mac0hi.Value16Low == 0xFFFF) || + (mac0lo.Value32 == 0x00000000 && mac0hi.Value16Low == 0x0000)) + { + TraceWrite("DevicePrepareHardware-Mac0-bad", LEVEL_WARNING, + TraceLoggingBinary(context->currentMacAddress, sizeof(context->currentMacAddress), "Mac0")); + context->currentMacAddress[0] = 0xF2; + context->currentMacAddress[1] = 0x00; + BCryptGenRandom( + nullptr, + context->currentMacAddress + 2, + sizeof(context->currentMacAddress) - 2, + BCRYPT_USE_SYSTEM_PREFERRED_RNG); + } + else if (context->currentMacAddress[0] & 1u) + { + TraceWrite("DevicePrepareHardware-Mac0-fixup", LEVEL_WARNING, + TraceLoggingBinary(context->currentMacAddress, sizeof(context->currentMacAddress), "Mac0")); + context->currentMacAddress[0] &= ~1u; + } + } + } + + // Read features + + { + auto const version = Read32(&context->regs->MacVersion); + context->feature0 = Read32(&context->regs->MacHwFeature0); + context->feature1 = Read32(&context->regs->MacHwFeature1); + context->feature2 = Read32(&context->regs->MacHwFeature2); + context->feature3 = Read32(&context->regs->MacHwFeature3); + TraceWrite("DevicePrepareHardware-config", LEVEL_INFO, + TraceLoggingHexInt32(version.RkVer, "RkVer"), + TraceLoggingHexInt32(version.UserVer, "UserVer"), + TraceLoggingHexInt32(context->feature0.Value32, "HwFeature0"), + TraceLoggingHexInt32(context->feature1.Value32, "HwFeature1"), + TraceLoggingHexInt32(context->feature2.Value32, "HwFeature2"), + TraceLoggingHexInt32(context->feature3.Value32, "HwFeature3"), + TraceLoggingBinary(context->permanentMacAddress, sizeof(context->permanentMacAddress), "PermanentAddr"), + TraceLoggingBinary(context->currentMacAddress, sizeof(context->currentMacAddress), "CurrentAddr")); + + if (version.RkVer < 0x51 || version.UserVer > 0x52) + { + TraceWrite("DevicePrepareHardware-RkVer-not-supported", LEVEL_ERROR, + TraceLoggingHexInt32(version.RkVer, "RkVer")); + status = STATUS_DEVICE_CONFIGURATION_ERROR; + goto Done; + } + } + + // Create DMA enabler + + { + auto const profile = context->feature1.AddressWidth == AddressWidth_32 + ? WdfDmaProfileScatterGather + : WdfDmaProfileScatterGather64; + WDF_DMA_ENABLER_CONFIG config; + WDF_DMA_ENABLER_CONFIG_INIT(&config, profile, 16384); // TODO: Jumbo packets. + config.WdmDmaVersionOverride = 3; + + switch (context->feature1.AddressWidth) + { + case AddressWidth_32: + config.AddressWidthOverride = 32; + maxPhysicalAddress.QuadPart = 0xFFFFFFFF; + break; + case AddressWidth_40: + config.AddressWidthOverride = 40; + maxPhysicalAddress.QuadPart = 0xFFFFFFFFFF; + break; + case AddressWidth_48: + config.AddressWidthOverride = 48; + maxPhysicalAddress.QuadPart = 0xFFFFFFFFFFFF; + break; + default: + TraceWrite("DevicePrepareHardware-AddressWidth-unknown", LEVEL_ERROR, + TraceLoggingHexInt32(context->feature1.AddressWidth, "AddressWidth")); + status = STATUS_DEVICE_CONFIGURATION_ERROR; + goto Done; + } + + status = WdfDmaEnablerCreate(device, &config, WDF_NO_OBJECT_ATTRIBUTES, &context->dma); + if (!NT_SUCCESS(status)) + { + TraceWrite("WdfDmaEnablerCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + } + + // Update adapter configuration. + + { + NET_ADAPTER_LINK_LAYER_ADDRESS address; + NET_ADAPTER_LINK_LAYER_ADDRESS_INIT(&address, sizeof(context->currentMacAddress), context->currentMacAddress); + NetAdapterSetCurrentLinkLayerAddress(context->adapter, &address); + NET_ADAPTER_LINK_LAYER_ADDRESS_INIT(&address, sizeof(context->permanentMacAddress), context->permanentMacAddress); + NetAdapterSetPermanentLinkLayerAddress(context->adapter, &address); + + NET_ADAPTER_LINK_STATE linkState; + NET_ADAPTER_LINK_STATE_INIT_DISCONNECTED(&linkState); + NetAdapterSetLinkState(context->adapter, &linkState); + + NET_ADAPTER_LINK_LAYER_CAPABILITIES linkCaps; + auto const maxSpeed = context->feature0.Gmii ? 1'000'000'000u : 100'000'000u; + NET_ADAPTER_LINK_LAYER_CAPABILITIES_INIT(&linkCaps, maxSpeed, maxSpeed); + NetAdapterSetLinkLayerCapabilities(context->adapter, &linkCaps); + + NetAdapterSetLinkLayerMtuSize(context->adapter, 1500); // TODO: Jumbo packets. + + NET_ADAPTER_DMA_CAPABILITIES dmaCaps; + NET_ADAPTER_DMA_CAPABILITIES_INIT(&dmaCaps, context->dma); + dmaCaps.MaximumPhysicalAddress = maxPhysicalAddress; + + NET_ADAPTER_TX_CAPABILITIES txCaps; + NET_ADAPTER_TX_CAPABILITIES_INIT_FOR_DMA(&txCaps, &dmaCaps, 1); + txCaps.MaximumNumberOfFragments = QueueDescriptorMinCount - 1; + + NET_ADAPTER_RX_CAPABILITIES rxCaps; // TODO: Might use less memory if driver-managed. + NET_ADAPTER_RX_CAPABILITIES_INIT_SYSTEM_MANAGED_DMA(&rxCaps, &dmaCaps, RxBufferSize, 1); // TODO: Jumbo packets. + + NetAdapterSetDataPathCapabilities(context->adapter, &txCaps, &rxCaps); + + // Note: If we don't claim support for everything, tcpip does not reliably bind. + NET_ADAPTER_RECEIVE_FILTER_CAPABILITIES rxFilterCaps; + NET_ADAPTER_RECEIVE_FILTER_CAPABILITIES_INIT(&rxFilterCaps, AdapterSetReceiveFilter); + rxFilterCaps.MaximumMulticastAddresses = + context->feature0.MacAddrCount > 1 + ? context->feature0.MacAddrCount - 1 + : 0; + rxFilterCaps.SupportedPacketFilters = + NetPacketFilterFlagDirected | + (rxFilterCaps.MaximumMulticastAddresses ? NetPacketFilterFlagMulticast : NET_PACKET_FILTER_FLAGS()) | + NetPacketFilterFlagAllMulticast | + NetPacketFilterFlagBroadcast | + NetPacketFilterFlagPromiscuous; + NetAdapterSetReceiveFilterCapabilities(context->adapter, &rxFilterCaps); + } + + // Initialize adapter. + + { + status = DeviceReset(context->regs, context->currentMacAddress); + if (!NT_SUCCESS(status)) + { + goto Done; + } + + // TODO: use ACPI _DSD? + // TODO: review. This is what the NetBSD driver seems to be doing, and + // it seems to work ok, but it doesn't line up with the documentation. + auto busMode = Read32(&context->regs->DmaSysBusMode); + busMode.Reserved14 = true; // mixed-burst? + busMode.FixedBurst = false; + busMode.AxiMaxWriteOutstanding = DefaultAxiMaxWriteOutstanding; + busMode.AxiMaxReadOutstanding = DefaultAxiMaxReadOutstanding; + busMode.BurstLength16 = true; + busMode.BurstLength8 = true; + busMode.BurstLength4 = true; + Write32(&context->regs->DmaSysBusMode, busMode); + + Write32(&context->regs->Mac1usTicCounter, DefaultCsrRate / 1'000'000u - 1); + + static_assert(sizeof(RxDescriptor) == sizeof(TxDescriptor)); + static_assert(sizeof(RxDescriptor) % BusBytes == 0, + "RxDescriptor must be a multiple of bus width."); + ChannelDmaControl_t dmaControl = {}; + dmaControl.DescriptorSkipLength = (sizeof(RxDescriptor) - 16) / BusBytes; + dmaControl.PblX8 = QueueBurstLengthX8; + Write32(&context->regs->DmaCh[0].DmaControl, dmaControl); + + Write32(&context->regs->MmcControl, 0x1); // Reset counters. + } + + // Start adapter. + + status = NetAdapterStart(context->adapter); + if (!NT_SUCCESS(status)) + { + TraceWrite("NetAdapterStart-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + +Done: + + TraceEntryExitWithStatus(DevicePrepareHardware, LEVEL_INFO, status); + return status; +} + +static EVT_WDF_DEVICE_RELEASE_HARDWARE DeviceReleaseHardware; +__declspec(code_seg("PAGE")) +static NTSTATUS +DeviceReleaseHardware( + _In_ WDFDEVICE device, + _In_ WDFCMRESLIST resourcesTranslated) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + UNREFERENCED_PARAMETER(resourcesTranslated); + + auto const context = DeviceGetContext(device); + if (context->regs != nullptr) + { +#define CtxStat(x) TraceLoggingUInt32(context->x, #x) +#define RegStat(x) TraceLoggingUInt32(Read32(&context->regs->x), #x) + + TraceWrite("DeviceReleaseHardware-MacStats", LEVEL_INFO, + CtxStat(isrHandled), + CtxStat(isrIgnored), + CtxStat(dpcLinkState), + CtxStat(dpcRx), + CtxStat(dpcTx), + CtxStat(dpcAbnormalStatus), + CtxStat(dpcFatalBusError)); + TraceWrite("DeviceReleaseHardware-TxStats", LEVEL_INFO, + RegStat(TxPacketCountGoodBad), + RegStat(TxUnderflowErrorPackets), + RegStat(TxCarrierErrorPackets), + RegStat(TxPacketCountGood), + RegStat(TxPausePackets)); + TraceWrite("DeviceReleaseHardware-RxStats", LEVEL_INFO, + RegStat(RxPacketCountGoodBad), + RegStat(RxCrcErrorPackets), + RegStat(RxLengthErrorPackets), + RegStat(RxPausePackets), + RegStat(RxFifoOverflowPackets), + RegStat(RxWatchdogErrorPackets)); + + DeviceReset(context->regs, context->permanentMacAddress); + MmUnmapIoSpace(context->regs, sizeof(*context->regs)); + context->regs = nullptr; + } + + TraceEntryExit(DeviceReleaseHardware, LEVEL_INFO); + return STATUS_SUCCESS; +} + +void +DeviceSetNotificationRxQueue( + _In_ NETADAPTER adapter, + _In_opt_ NETPACKETQUEUE rxQueue) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = DeviceGetContext(AdapterGetContext(adapter)->device); + + WdfSpinLockAcquire(context->lock); + + context->rxQueue = rxQueue; + if (rxQueue) + { + DeviceInterruptEnable_Locked(context, InterruptsRx); + } + else + { + DeviceInterruptDisable_Locked(context, InterruptsRx); + } + + WdfSpinLockRelease(context->lock); +} + +void +DeviceSetNotificationTxQueue( + _In_ NETADAPTER adapter, + _In_opt_ NETPACKETQUEUE txQueue) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = DeviceGetContext(AdapterGetContext(adapter)->device); + + WdfSpinLockAcquire(context->lock); + + context->txQueue = txQueue; + if (txQueue) + { + DeviceInterruptEnable_Locked(context, InterruptsTx); + } + else + { + DeviceInterruptDisable_Locked(context, InterruptsTx); + } + + WdfSpinLockRelease(context->lock); +} + +__declspec(code_seg("PAGE")) +NTSTATUS +DeviceAdd( + _In_ WDFDRIVER driver, + _Inout_ PWDFDEVICE_INIT deviceInit) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + UNREFERENCED_PARAMETER(driver); + + NTSTATUS status; + + // Configure deviceInit + + status = NetDeviceInitConfig(deviceInit); + if (!NT_SUCCESS(status)) + { + TraceWrite("NetDeviceInitConfig-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + + { + WDF_PNPPOWER_EVENT_CALLBACKS pnpPowerCallbacks; + WDF_PNPPOWER_EVENT_CALLBACKS_INIT(&pnpPowerCallbacks); + pnpPowerCallbacks.EvtDevicePrepareHardware = DevicePrepareHardware; + pnpPowerCallbacks.EvtDeviceReleaseHardware = DeviceReleaseHardware; + pnpPowerCallbacks.EvtDeviceD0Entry = DeviceD0Entry; + pnpPowerCallbacks.EvtDeviceD0Exit = DeviceD0Exit; + WdfDeviceInitSetPnpPowerEventCallbacks(deviceInit, &pnpPowerCallbacks); + } + + // Create device. + + WDFDEVICE device; + { + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, DeviceContext); + + status = WdfDeviceCreate(&deviceInit, &attributes, &device); + if (!NT_SUCCESS(status)) + { + TraceWrite("WdfDeviceCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + + WdfDeviceSetAlignmentRequirement(device, FILE_BYTE_ALIGNMENT); + + WDF_DEVICE_STATE deviceState; + WDF_DEVICE_STATE_INIT(&deviceState); + deviceState.NotDisableable = WdfFalse; + WdfDeviceSetDeviceState(device, &deviceState); + } + + // Create lock. + + { + auto const context = DeviceGetContext(device); + + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT(&attributes); + attributes.ParentObject = device; + + status = WdfSpinLockCreate(&attributes, &context->lock); + if (!NT_SUCCESS(status)) + { + TraceWrite("WdfSpinLockCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + } + + // Create adapter. + + { + auto const context = DeviceGetContext(device); + auto const adapterInit = NetAdapterInitAllocate(device); + if (adapterInit == nullptr) + { + TraceWrite("NetAdapterInitAllocate-failed", LEVEL_ERROR); + status = STATUS_INSUFFICIENT_RESOURCES; + goto Done; + } + + NET_ADAPTER_DATAPATH_CALLBACKS callbacks; + NET_ADAPTER_DATAPATH_CALLBACKS_INIT(&callbacks, AdapterCreateTxQueue, AdapterCreateRxQueue); + NetAdapterInitSetDatapathCallbacks(adapterInit, &callbacks); + + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, AdapterContext); + status = NetAdapterCreate(adapterInit, &attributes, &context->adapter); + NetAdapterInitFree(adapterInit); + if (!NT_SUCCESS(status)) + { + TraceWrite("NetAdapterCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + + auto const adapterContext = AdapterGetContext(context->adapter); + NT_ASSERT(adapterContext->device == nullptr); + adapterContext->device = device; + } + +Done: + + TraceEntryExitWithStatus(DeviceAdd, LEVEL_INFO, status); + return status; +} diff --git a/drivers/net/dwc_eqos/device.h b/drivers/net/dwc_eqos/device.h new file mode 100644 index 0000000..98fd5af --- /dev/null +++ b/drivers/net/dwc_eqos/device.h @@ -0,0 +1,15 @@ +#pragma once + +void +DeviceSetNotificationRxQueue( + _In_ NETADAPTER adapter, + _In_opt_ NETPACKETQUEUE rxQueue); + +void +DeviceSetNotificationTxQueue( + _In_ NETADAPTER adapter, + _In_opt_ NETPACKETQUEUE txQueue); + +__declspec(code_seg("PAGE")) +EVT_WDF_DRIVER_DEVICE_ADD +DeviceAdd; diff --git a/drivers/net/dwc_eqos/driver.cpp b/drivers/net/dwc_eqos/driver.cpp new file mode 100644 index 0000000..f36ef04 --- /dev/null +++ b/drivers/net/dwc_eqos/driver.cpp @@ -0,0 +1,74 @@ +#include "precomp.h" +#include "device.h" +#include "trace.h" + +/* +TODO list: +- Support for 10/100 Mbps modes (requires ACPI support). +- Use ACPI to get DMA configuration. +- Jumbo frames. +- Receive queue memory optimization. +- Configuration. +- Checksum offload. +- Tx segmentation offload. +- Rx segmentation offload. +- Wake-on-LAN. +- ARP offload. +- Multi-queue support. +*/ + +TRACELOGGING_DEFINE_PROVIDER( + TraceProvider, + "dwc_eqos", + // *dwc_eqos = {5d8331d3-70b3-5620-5664-db28f48a4b79} + (0x5d8331d3, 0x70b3, 0x5620, 0x56, 0x64, 0xdb, 0x28, 0xf4, 0x8a, 0x4b, 0x79)); + +static EVT_WDF_DRIVER_UNLOAD DriverUnload; +__declspec(code_seg("PAGE")) +void +DriverUnload(_In_ WDFDRIVER driver) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + UNREFERENCED_PARAMETER(driver); + TraceEntryExit(DriverUnload, LEVEL_INFO); + TraceLoggingUnregister(TraceProvider); +} + +extern "C" DRIVER_INITIALIZE DriverEntry; +__declspec(code_seg("INIT")) +NTSTATUS +DriverEntry( + _In_ DRIVER_OBJECT* driverObject, + _In_ UNICODE_STRING* registryPath) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + + NTSTATUS status; + + TraceLoggingRegister(TraceProvider); + TraceEntry(DriverEntry, LEVEL_INFO, + TraceLoggingUnicodeString(registryPath)); + + WDF_DRIVER_CONFIG config; + WDF_DRIVER_CONFIG_INIT(&config, DeviceAdd); + config.EvtDriverUnload = DriverUnload; + config.DriverPoolTag = 'dwcE'; + + status = WdfDriverCreate( + driverObject, + registryPath, + WDF_NO_OBJECT_ATTRIBUTES, + &config, + WDF_NO_HANDLE); + + TraceExitWithStatus(DriverEntry, LEVEL_INFO, status); + + if (!NT_SUCCESS(status)) + { + TraceLoggingUnregister(TraceProvider); + } + + return status; +} diff --git a/drivers/net/dwc_eqos/dwc_eqos.inf b/drivers/net/dwc_eqos/dwc_eqos.inf new file mode 100644 index 0000000..7ce273f --- /dev/null +++ b/drivers/net/dwc_eqos/dwc_eqos.inf @@ -0,0 +1,121 @@ +; MIT License +; +; Copyright (c) Doug Cook. +; +; Permission is hereby granted, free of charge, to any person obtaining a copy +; of this software and associated documentation files (the "Software"), to deal +; in the Software without restriction, including without limitation the rights +; to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +; copies of the Software, and to permit persons to whom the Software is +; furnished to do so, subject to the following conditions: +; +; The above copyright notice and this permission notice shall be included in all +; copies or substantial portions of the Software. +; +; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +; AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +; LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +; SOFTWARE + +[Version] +Signature = "$WINDOWS NT$" +Class = Net +ClassGuid = {4d36e972-e325-11ce-bfc1-08002be10318} +Provider = %ProviderName% +PnpLockdown = 1 +DriverVer = ; will be filled in at build time by StampInf +CatalogFile = dwc_eqos.cat + +[ControlFlags] +ExcludeFromSelect = * + +[SourceDisksNames] +1 = Disk + +[SourceDisksFiles] +dwc_eqos.sys = 1 + +[DestinationDirs] +DefaultDestDir = 12 + +[Manufacturer] +; Windows 10 1709 16299: KMDF 1.23; NetAdapterCx 2.0 - NetAdapterCx preview?, no packet filter control. +; Windows 10 2004 19041: KMDF 1.31; NetAdapterCx 2.0 - MBBCx supported, Ethernet preview, no packet filter control. +; Windows 10 PRVW 19536: KMDF 1.31; NetAdapterCx 2.1 - Unsupported, adds packet filter control. +; Windows 11 21H2 22000: KMDF 1.33; NetAdapterCx 2.2 - Ethernet supported, packet filter control supported. +%RKCP%=DWC,NT$ARCH$.10.0...19536 + +[DWC.NT$ARCH$.10.0...19536] +%DWCEQOS.DeviceDesc% = DWCEQOS_Device, ACPI\RKCP6543 + +[DWCEQOS_Device.NT] +CopyFiles = DWCEQOS_CopyFiles +AddReg = DWCEQOS_AddReg_Ndi +AddReg = DWCEQOS_AddReg_Ndi_params +FeatureScore = 0xFE +Characteristics = 0x04 ; NCF_PHYSICAL +BusType = 15 ; PNPBus +*IfType = 6 ; IF_TYPE_ETHERNET_CSMACD +*MediaType = 0 ; NdisMedium802_3 +*PhysicalMediaType = 14 ; NdisPhysicalMedium802_3 +*IfConnectorPresent = 1 +*ConnectionType = 1 ; NET_IF_CONNECTION_DEDICATED +*DirectionType = 0 ; NET_IF_DIRECTION_SENDRECEIVE +*AccessType = 2 ; NET_IF_ACCESS_BROADCAST +*HardwareLoopback = 0 + +[DWCEQOS_CopyFiles] +dwc_eqos.sys + +[DWCEQOS_AddReg_Ndi] +HKR, Ndi, Service, 0, %ServiceName% +HKR, Ndi\Interfaces, UpperRange, 0, "ndis5" +HKR, Ndi\Interfaces, LowerRange, 0, "ethernet" + +[DWCEQOS_AddReg_Ndi_params] +HKR, Ndi\params\NetworkAddress, ParamDesc, 0, %NetworkAddress% +HKR, Ndi\params\NetworkAddress, type, 0, "edit" +HKR, Ndi\params\NetworkAddress, default, 0, "" +HKR, Ndi\params\NetworkAddress, LimitText, 0, "12" +HKR, Ndi\params\NetworkAddress, UpperCase, 0, "1" +HKR, Ndi\params\NetworkAddress, Optional, 0, "1" + +[DWCEQOS_Device.NT.Services] +AddService = %ServiceName%, 2, DWCEQOS_AddService, DWCEQOS_AddService_EventLog + +[DWCEQOS_AddService] +AddReg = DWCEQOS_AddService.AddReg +DisplayName = %DWCEQOS.ServiceDesc% +ServiceType = 1 ; SERVICE_KERNEL_DRIVER +StartType = 3 ; SERVICE_DEMAND_START +ErrorControl = 1 ; SERVICE_ERROR_NORMAL +ServiceBinary = %12%\dwc_eqos.sys + +[DWCEQOS_AddService.AddReg] +HKR,, BootFlags, 0x10001, 1 ; CM_SERVICE_NETWORK_BOOT_LOAD + +[DWCEQOS_AddService_EventLog] +AddReg = DWCEQOS_AddService_EventLog.AddReg + +[DWCEQOS_AddService_EventLog.AddReg] +HKR,, EventMessageFile, 0x00020000, "%%SystemRoot%%\System32\netevent.dll" +HKR,, TypesSupported, 0x00010001, 7 + +[DWCEQOS_Device.NT.Wdf] +KmdfService = %ServiceName%, DWCEQOS_KmdfService + +[DWCEQOS_KmdfService] +KmdfLibraryVersion = $KMDFVERSION$ + +[Strings] +ProviderName = "Open Source" +NetworkAddress = "Network Address" +RKCP = "Rockchip" +DWCEQOS.DeviceDesc = "Synopsys DesignWare Ethernet Quality of Service (GMAC)" +DWCEQOS.ServiceDesc = "DesignWare Ethernet" + +; Not localized +ServiceName = "dwc_eqos" diff --git a/drivers/net/dwc_eqos/dwc_eqos.vcxproj b/drivers/net/dwc_eqos/dwc_eqos.vcxproj new file mode 100644 index 0000000..c2941c9 --- /dev/null +++ b/drivers/net/dwc_eqos/dwc_eqos.vcxproj @@ -0,0 +1,121 @@ + + + + + Debug + ARM64 + + + Release + ARM64 + + + + + + + + + + Create + Create + + + + + + + + + + + + + + + + {CAD465B6-97D7-402D-85F8-50DE140A6D9D} + $(MSBuildProjectName) + Debug + ARM64 + {497e31cb-056b-4f31-abb8-447fd55ee5a5} + $(LatestTargetPlatformVersion) + + + + Windows10 + true + WindowsKernelModeDriver10.0 + Driver + KMDF + Universal + true + 2 + 1 + 1 + 31 + + + Windows10 + false + WindowsKernelModeDriver10.0 + Driver + KMDF + Universal + true + 2 + 1 + 1 + 31 + + + + + + + + + + + DbgengKernelDebugger + $(SolutionDir)$(Platform)\$(Configuration)\Output\ + $(SolutionDir)$(Platform)\$(Configuration)\Intermediate\$(TargetName)\ + + + DbgengKernelDebugger + $(SolutionDir)$(Platform)\$(Configuration)\Output\ + $(SolutionDir)$(Platform)\$(Configuration)\Intermediate\$(TargetName)\ + + + + Use + precomp.h + stdcpp20 + + + sha256 + + + $(DDK_LIB_PATH)Ksecdd.lib;%(AdditionalDependencies) + + + + + Use + precomp.h + stdcpp20 + + + sha256 + + + $(DDK_LIB_PATH)Ksecdd.lib;%(AdditionalDependencies) + + + + + + + + + \ No newline at end of file diff --git a/drivers/net/dwc_eqos/dwc_eqos.vcxproj.filters b/drivers/net/dwc_eqos/dwc_eqos.vcxproj.filters new file mode 100644 index 0000000..cae6be1 --- /dev/null +++ b/drivers/net/dwc_eqos/dwc_eqos.vcxproj.filters @@ -0,0 +1,69 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {8E41214B-6785-4CFE-B992-037D68949A14} + inf;inv;inx;mof;mc; + + + + + Driver Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/drivers/net/dwc_eqos/precomp.h b/drivers/net/dwc_eqos/precomp.h new file mode 100644 index 0000000..2960475 --- /dev/null +++ b/drivers/net/dwc_eqos/precomp.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +#pragma warning(push) +#pragma warning(disable: 4471) // forward declaration of an unscoped enumeration +#include +#pragma warning(pop) + +#include +#include +#include +#include +#include +#include diff --git a/drivers/net/dwc_eqos/queue_common.cpp b/drivers/net/dwc_eqos/queue_common.cpp new file mode 100644 index 0000000..e5e1ee7 --- /dev/null +++ b/drivers/net/dwc_eqos/queue_common.cpp @@ -0,0 +1,37 @@ +#include "precomp.h" +#include "queue_common.h" + +static_assert((QueueDescriptorAlignment & (QueueDescriptorAlignment - 1)) == 0, + "QueueDescriptorAlignment must be a power of 2."); + +_Use_decl_annotations_ UINT32 +QueueDescriptorCount(UINT32 fragmentCount) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto clamped = + fragmentCount < QueueDescriptorMinCount ? QueueDescriptorMinCount : + fragmentCount > QueueDescriptorMaxCount ? QueueDescriptorMaxCount : + fragmentCount; + + // Round up to a power of 2. + ULONG bits; + _BitScanReverse(&bits, clamped - 1); + auto const count = 2u << bits; + NT_ASSERT(count >= QueueDescriptorMinCount); + NT_ASSERT(count <= QueueDescriptorMaxCount); + NT_ASSERT((count & (count - 1)) == 0); + + return count; +} + +_Use_decl_annotations_ UINT32 +QueueDescriptorAddressToIndex(UINT32 address, PHYSICAL_ADDRESS descPhysical, UINT32 descCount) +{ + // DISPATCH_LEVEL + UNREFERENCED_PARAMETER(descPhysical); + UNREFERENCED_PARAMETER(descCount); + NT_ASSERT(address == 0 || address >= descPhysical.LowPart); + NT_ASSERT(address < descPhysical.LowPart + descCount * QueueDescriptorSize); + NT_ASSERT(address % QueueDescriptorSize == 0); + return (address & (QueueDescriptorAlignment - 1)) / QueueDescriptorSize; +} diff --git a/drivers/net/dwc_eqos/queue_common.h b/drivers/net/dwc_eqos/queue_common.h new file mode 100644 index 0000000..71c7745 --- /dev/null +++ b/drivers/net/dwc_eqos/queue_common.h @@ -0,0 +1,20 @@ +#pragma once + +UINT32 constexpr QueueDescriptorSize = 64; +UINT32 constexpr QueueDescriptorMinCount = PAGE_SIZE / QueueDescriptorSize; +UINT32 constexpr QueueDescriptorMaxCount = 0x400; + +bool constexpr QueueBurstLengthX8 = true; +UINT32 constexpr QueueBurstLength = 64u / (QueueBurstLengthX8 ? 8 : 1); // TODO: load from ACPI? + +// Alignment is mainly to make sure the allocation does not cross a 4GB boundary, +// but it also simplifies working with the addresses. +auto const QueueDescriptorAlignment = QueueDescriptorMaxCount * QueueDescriptorSize; + +_IRQL_requires_max_(PASSIVE_LEVEL) +UINT32 +QueueDescriptorCount(UINT32 fragmentCount); + +_IRQL_requires_max_(DISPATCH_LEVEL) +UINT32 +QueueDescriptorAddressToIndex(UINT32 address, PHYSICAL_ADDRESS descPhysical, UINT32 descCount); diff --git a/drivers/net/dwc_eqos/registers.h b/drivers/net/dwc_eqos/registers.h new file mode 100644 index 0000000..9bd78e1 --- /dev/null +++ b/drivers/net/dwc_eqos/registers.h @@ -0,0 +1,2024 @@ +#pragma once +#pragma warning(push) +#pragma warning(disable:4201) // nonstandard extension used: nameless struct/union + +#pragma region Read32/Write32 + +template +struct TypeIdentity { using type = T; }; + +template +_IRQL_requires_max_(HIGH_LEVEL) +static T FORCEINLINE +Read32(_In_ T const* reg) +{ + // HIGH_LEVEL + static_assert(sizeof(T) == sizeof(ULONG)); + auto const val = READ_REGISTER_ULONG((ULONG*)reg); + return reinterpret_cast(val); +} + +template +_IRQL_requires_max_(HIGH_LEVEL) +static void FORCEINLINE +Write32(_Out_ T* reg, typename TypeIdentity::type val) +{ + // HIGH_LEVEL + static_assert(sizeof(T) == sizeof(ULONG)); +#pragma warning(suppress: 6001) // WRITE_REGISTER_ULONG is not annotated correctly. + WRITE_REGISTER_ULONG(reinterpret_cast(reg), reinterpret_cast(val)); +} + +#pragma endregion + +// Adapted from Rockchip RK3588 TRM V1.0. + +#pragma region ChannelRegisters + +union ChannelDmaControl_t +{ + UINT32 Value32; + struct + { + UINT32 MaxSegmentSize : 14; // MSS - Maximum Segment Size + UINT32 Reserved14 : 2; + + UINT32 PblX8 : 1; // PBLx8 - 8xPBL Mode + UINT32 Reserved17 : 1; + UINT32 DescriptorSkipLength : 3; // DSL - Descriptor Skip Length + UINT32 Reserved21 : 11; + }; +}; + +union ChannelTxControl_t +{ + UINT32 Value32; + struct + { + UINT32 Start : 1; // ST + UINT32 Reserved1 : 3; + UINT32 OperateOnSecondPacket : 1; // OSF + UINT32 Reserved5 : 7; + UINT32 TcpSegmentation : 1; // TSE + UINT32 TcpSegmentationMode : 2; // TSE_MODE + UINT32 IgnorePbl : 1; // IPBL + + UINT32 TxPbl : 6; // TxPBL - Transmit Programmable Burst Length + UINT32 Reserved22 : 10; + }; +}; + +union ChannelRxControl_t +{ + UINT32 Value32; + struct + { + UINT32 Start : 1; // SR + UINT32 ReceiveBufferSize : 14; // RBSZ - Receive Buffer Size, low 3 bits must be 0. + UINT32 Reserved15 : 1; + + UINT32 RxPbl : 6; // RxPBL - Receive Programmable Burst Length + UINT32 Reserved22 : 9; + UINT32 RxPacketFlush : 1; // RPF - Receive Packet Flush + }; +}; + +union ChannelInterruptEnable_t +{ + UINT32 Value32; + struct + { + UINT32 Tx : 1; // TIE + UINT32 TxStopped : 1; // TXSE + UINT32 TxBufferUnavailable : 1; // TBUE + UINT32 Reserved3 : 3; + UINT32 Rx : 1; // RIE + UINT32 RxBufferUnavailable : 1; // RBUE + + UINT32 RxStopped : 1; // RSE + UINT32 RxWatchdogTimeout : 1; // RWTE + UINT32 EarlyTx : 1; // ETIE + UINT32 EarlyRx : 1; // ERIE + UINT32 FatalBusError : 1; // FBEE + UINT32 ContextDescriptorError : 1; // CDEE + UINT32 AbnormalInterruptSummary : 1; // AIE + UINT32 NormalInterruptSummary : 1; // NIE + + UINT32 Reserved16 : 16; + }; +}; + +enum ChannelDmaError_t : UINT32 +{ + ChannelDmaError_WriteBufferDmaOk = 0, // 000 + ChannelDmaError_ReadBufferDmaOk, // 001 + ChannelDmaError_WriteDescriptorDmaOk, // 010 + ChannelDmaError_ReadDescriptorDmaOk, // 011 + ChannelDmaError_WriteBufferDmaError, // 100 + ChannelDmaError_ReadBufferDmaError, // 101 + ChannelDmaError_WriteDescriptorDmaError, // 110 + ChannelDmaError_ReadDescriptorDmaError, // 111 +}; + +union ChannelStatus_t +{ + UINT32 Value32; + struct + { + UINT32 Tx : 1; // TI + UINT32 TxStopped : 1; // TPS + UINT32 TxBufferUnavailable : 1; // TBU + UINT32 Reserved3 : 3; + UINT32 Rx : 1; // RI + UINT32 RxBufferUnavailable : 1; // RBU + + UINT32 RxStopped : 1; // RPS + UINT32 RxWatchdogTimeout : 1; // RWT + UINT32 EarlyTx : 1; // ETI + UINT32 EarlyRx : 1; // ERI + UINT32 FatalBusError : 1; // FBE + UINT32 ContextDescriptorError : 1; // CDE + UINT32 AbnormalInterruptSummary : 1; // AIS + UINT32 NormalInterruptSummary : 1; // NIS + + ChannelDmaError_t TxDmaError : 3; // TEB - valid only when the FBE bit is set + ChannelDmaError_t RxDmaError : 3; // REB - valid only when the FBE bit is set + UINT32 Reserved22 : 10; + }; +}; + +struct ChannelRegisters +{ + // DMA_CHx_Control @ 0x00 = 0x0: + // The register specifies the MSS value for segmentation, length to skip between + // two descriptors, and 8xPBL mode. + ChannelDmaControl_t DmaControl; + + // DMA_CHx_Tx_Control @ 0x04 = 0x0: + // The register controls the Tx features such as PBL, TCP segmentation, and Tx + // Channel weights. + ChannelTxControl_t TxControl; + + // DMA_CHx_Rx_Control @ 0x08 = 0x0: + // The DMA Channel0 Receive Control register controls the Rx features such as PBL, + // buffer size, and extended status. + ChannelRxControl_t RxControl; + + ULONG Padding0C[1]; + + // DMA_CHx_TxDesc_List_Address_Hi @ 0x10 = 0x0: + ULONG TxDescListAddressHigh; + + // DMA_CHx_TxDesc_List_Address @ 0x14 = 0x0: + // The Channel0 Tx Descriptor List Address register points the DMA to the start of + // Transmit. + ULONG TxDescListAddress; + + // DMA_CHx_RxDesc_List_Address_Hi @ 0x18 = 0x0: + ULONG RxDescListAddressHigh; + + // DMA_CHx_RxDesc_List_Address @ 0x1C = 0x0: + // The Channel0 Rx Descriptor List Address register points the DMA to the start of + // Receive descriptor list. + ULONG RxDescListAddress; + + // DMA_CHx_TxDesc_Tail_Pointer @ 0x20 = 0x0: + // The Channel0 Tx Descriptor Tail Pointer register points to an offset from the + // base and indicates the location of the last valid descriptor. + ULONG TxDescTailPointer; + + ULONG Padding24[1]; + + // DMA_CHx_RxDesc_Tail_Pointer @ 0x28 = 0x0: + // The Channel0 Rx Descriptor Tail Pointer Points to an offset from the base and + // indicates the location of the last valid descriptor. + ULONG RxDescTailPointer; + + // DMA_CHx_TxDesc_Ring_Length @ 0x2C = 0x0: + // The Tx Descriptor Ring Length register contains the length of the Transmit + // descriptor ring. + ULONG TxDescRingLength; + + // DMA_CHx_RxDesc_Ring_Length @ 0x30 = 0x0: + // The Channel0 Rx Descriptor Ring Length register contains the length of the + // Receive descriptor circular ring. + ULONG RxDescRingLength; + + // DMA_CHx_Interrupt_Enable @ 0x34 = 0x0: + // The Channel0 Interrupt Enable register enables the interrupts reported by the + // Status register. + ChannelInterruptEnable_t InterruptEnable; + + // DMA_CHx_Rx_Interrupt_WD_Timer @ 0x38 = 0x0: + // The Receive Interrupt Watchdog Timer register indicates the watchdog timeout + // for Receive Interrupt (RI) from the DMA. + ULONG RxInterruptWdTimer; + + // DMA_CHx_Slot_Func_Ctrl_Status @ 0x3C = 0x7C0: + // The Slot Function Control and Status register contains the control bits for + // slot function and the status for Transmit path. + ULONG SlotFuncCtrlStatus; + + ULONG Padding40[1]; + + // DMA_CHx_Current_App_TxDesc @ 0x44 = 0x0: + // The Channel0 Current Application Transmit Descriptor register points to the + // current Transmit descriptor read by the DMA. + ULONG TxCurrentAppDesc; + + ULONG Padding48[1]; + + // DMA_CHx_Current_App_RxDesc @ 0x4C = 0x0: + // The Channel0 Current Application Receive Descriptor register points to the + // current Receive descriptor read by the DMA. + ULONG RxCurrentAppDesc; + + ULONG Padding50[1]; + + // DMA_CHx_Current_App_TxBuffer @ 0x54 = 0x0: + // The Channel0 Current Application Transmit Buffer Address register points to the + // current Tx buffer address read by the DMA. + ULONG TxCurrentAppBuffer; + + ULONG Padding58[1]; + + // DMA_CHx_Current_App_RxBuffer @ 0x5C = 0x0: + // The Channel0 Current Application Receive Buffer Address register points to the + // current Rx buffer address read by the DMA. + ULONG RxCurrentAppBuffer; + + // DMA_CHx_Status @ 0x60 = 0x0: + // The software driver (application) reads the Status register during interrupt + // service routine or polling to determine the status of the DMA. + ChannelStatus_t Status; + + // DMA_CHx_Miss_Frame_Cnt @ 0x64 = 0x0: + // This register has the number of packet counter that got dropped by the DMA + // either due to Bus Error or due to programming RPF field in RxControl + // register. + ULONG MissFrameCnt; + + // DMA_CHx_RX_ERI_Cnt @ 0x68 = 0x0: + // The RxEriCnt registers provides the count of the number of times ERI + // was asserted. + ULONG RxEriCnt; + + ULONG Padding6C[5]; +}; +static_assert(sizeof(ChannelRegisters) == 128); + +#pragma endregion + +#pragma region MtlQueueRegisters + +enum MtlTxQueueEnable_t : UINT32 +{ + MtlTxQueueEnable_Disabled = 0, + MtlTxQueueEnable_EnabledAV, + MtlTxQueueEnable_Enabled, +}; + +enum MtlTxReadStatus_t : UINT32 +{ + MtlTxReadStatus_Idle = 0, + MtlTxReadStatus_Read, + MtlTxReadStatus_Waiting, + MtlTxReadStatus_Flushing, +}; + +union MtlTxOperationMode_t +{ + UINT32 Value32; + struct + { + UINT32 FlushTxQueue : 1; // FTQ + UINT32 StoreAndForward : 1; // TSF + MtlTxQueueEnable_t QueueEnable : 2; // TXQEN + UINT32 ThresholdControl : 3; // TTC + UINT32 Reserved7 : 9; + UINT32 QueueSize : 6; // TQS + UINT32 Reserved22 : 10; + }; +}; + +union MtlTxDebug_t +{ + UINT32 Value32; + struct + { + UINT32 Paused : 1; // TXQPAUSED + MtlTxReadStatus_t ReadStatus : 2; // TRCSTS + UINT32 WriteStatus : 1; // TWCSTS + UINT32 NotEmpty : 1; // TXQSTS + UINT32 FifoFull : 1; // TXSTSFSTS + UINT32 Reserved6 : 10; + + UINT32 QueuePackets : 3; // PTXQ + UINT32 Reserved19 : 1; + UINT32 QueueStatusWords : 3; // STXSTSF + UINT32 Reserved23 : 9; + }; +}; + +union MtlRxOperationMode_t +{ + UINT32 Value32; + struct + { + UINT32 ThresholdControl : 2; // RTC + UINT32 Reserved2 : 1; + UINT32 ForwardUndersizedGoodPackets : 1; // FUP + UINT32 ForwardErrorPackets : 1; // FEP + UINT32 StoreAndForward : 1; // RSF + UINT32 DisableDropTcpChecksumError : 1; // DIS_TCP_EF + UINT32 HardwareFlowControl : 1; // EHFC + UINT32 FlowControlActivate : 6; // RFA + UINT32 FlowControlDeactivate : 6; // RFD + UINT32 QueueSize : 7; // RQS + UINT32 Reserved27 : 5; + }; +}; + +struct MtlQueueRegisters +{ + // MTL_TxQx_Operation_Mode @ 0x00 = 0x60000: + // The Queue X Transmit Operation Mode register establishes the Transmit queue + // operating modes and commands. + MtlTxOperationMode_t TxOperationMode; + + // MTL_TxQx_Underflow @ 0x04 = 0x0: + // The Queue X Underflow Counter register contains the counter for packets aborted + // because of Transmit queue underflow and packets missed because of Receive queue + // packet flush. + ULONG TxUnderflow; + + // MTL_TxQx_Debug @ 0x08 = 0x0: + // The Queue X Transmit Debug register gives the debug status of various blocks + // related to the Transmit queue. + MtlTxDebug_t TxDebug; + + ULONG Padding0C[2]; + + // MTL_TxQx_ETS_Status @ 0x14 = 0x0: + // The Queue X ETS Status register provides the average traffic transmitted in + // Queue X. + ULONG TxEtsStatus; + + // MTL_TxQx_Quantum_Weight @ 0x18 = 0x0: + // The Queue X Quantum or Weights register contains the quantum value for Deficit + // Weighted Round Robin (DWRR), weights for the Weighted Round Robin (WRR), and + // Weighted Fair Queuing (WFQ) for Queue X. + ULONG TxQuantumWeight; + + ULONG Padding1C[4]; + + // MTL_Qx_Interrupt_Ctrl_Status @ 0x2C = 0x0: + // This register contains the interrupt enable and status bits for the queue X + // interrupts. + ULONG InterruptCtrlStatus; + + // MTL_RxQx_Operation_Mode @ 0x30 = 0x0: + // The Queue X Receive Operation Mode register establishes the Receive queue + // operating modes and command. + MtlRxOperationMode_t RxOperationMode; + + // MTL_RxQx_Miss_Pkt_Ovf_Cnt @ 0x34 = 0x0: + // The Queue X Missed Packet and Overflow Counter register contains the counter + // for packets missed because of Receive queue packet flush and packets discarded + // because of Receive queue overflow. + ULONG RxMissPktOvfCnt; + + // MTL_RxQx_Debug @ 0x38 = 0x0: + // The Queue X Receive Debug register gives the debug status of various blocks + // related to the Receive queue. + ULONG RxDebug; + + // MTL_RxQx_Control @ 0x3C = 0x0: + // The Queue Receive Control register controls the receive arbitration and passing + // of received packets to the application. + ULONG RxControl; +}; +static_assert(sizeof(MtlQueueRegisters) == 64); + +#pragma endregion + +#pragma region MacAddressRegisters + +union MacAddressHigh_t +{ + UINT32 Value32; + struct + { + UINT16 Value16Low; + UINT16 Value16High; + }; + struct + { + BYTE Addr4; + BYTE Addr5; + BYTE DmaChannelSelect; + BYTE MaskByteControl : 6; + BYTE SourceAddress : 1; + BYTE AddressEnable : 1; + }; +}; + +union MacAddressLow_t +{ + UINT32 Value32; + struct + { + BYTE Addr0; + BYTE Addr1; + BYTE Addr2; + BYTE Addr3; + }; +}; + +struct MacAddressRegisters +{ + // MAC_AddressX_High @ 0x00 = 0xFFFF: + // The MAC AddressX High register holds the upper 16 bits of the Xth 6-byte MAC + // address of the station. + MacAddressHigh_t High; + + // MAC_AddressX_Low @ 0x04 = 0xFFFFFFFF: + // The MAC AddressX Low register holds the lower 32 bits of the Xth 6-byte MAC + // address of the station. + MacAddressLow_t Low; +}; +static_assert(sizeof(MacAddressRegisters) == 8); + +#pragma endregion + +#pragma region MacL3L4Registers + +struct MacL3L4Registers +{ + // MAC_L3_L4_ControlX @ 0x00 = 0x0: + // The Layer 3 and Layer 4 Control register controls the operations of filter X of + // Layer 3 and Layer 4. + ULONG Control; + + // MAC_Layer4_AddressX @ 0x04 = 0x0: + // The MAC_Layer4_Address, MAC_L3_L4_Control, MAC_Layer3_Addr0_Reg, + // MAC_Layer3_Addr1_Reg, MAC_Layer3_Addr2_Reg and MAC_Layer3_Addr3_Reg + // registers are reserved (RO with default value) if Enable Layer 3 and Layer 4 + // Packet Filter option is not selected while configuring the core. + ULONG Layer4Address; + + ULONG Padding08[2]; + + // MAC_Layer3_Addr0_RegX @ 0x10 = 0x0: + // For IPv4 packets, the Layer 3 Address 0 Register contains the 32-bit + // IP Source Address field. For IPv6 packets, it contains Bits[31:0] of the + // 128-bit IP Source Address or Destination Address field. + ULONG Layer3Addr0; + + // MAC_Layer3_Addr1_Reg @ 0x14 = 0x0: + // For IPv4 packets, the Layer 3 Address 1 Register contains the 32-bit + // IP Destination Address field. For IPv6 packets, it contains Bits[63:32] of the + // 128-bit IP Source Address or Destination Address field. + ULONG Layer3Addr1; + + // MAC_Layer3_Addr2_Reg @ 0x18 = 0x0: + // The Layer 3 Address 2 Register is reserved for IPv4 packets. For + // IPv6 packets, it contains Bits[95:64] of 128-bit IP Source Address or + // Destination Address field. + ULONG Layer3Addr2; + + // MAC_Layer3_Addr3_Reg @ 0x1C = 0x0: + // The Layer 3 Address 3 Register is reserved for IPv4 packets. For + // IPv6 packets, it contains Bits[127:96] of 128-bit IP Source Address or + // Destination Address field. + ULONG Layer3Addr3; + + ULONG Padding20[4]; +}; +static_assert(sizeof(MacL3L4Registers) == 48); + +#pragma endregion + +#pragma region MacRegisters + +enum PortSelectSpeed_t : UINT32 +{ + PortSelectSpeed_1000M=0,// PS = 0, FES = 0 + PortSelectSpeed_2500M, // PS = 0, FES = 1 + PortSelectSpeed_10M, // PS = 1, FES = 0 + PortSelectSpeed_100M, // PS = 1, FES = 1 +}; + +enum PhyIfSpeed_t : UINT32 +{ + PhyIfSpeed_2_5M = 0, // 2.5 MHz + PhyIfSpeed_25M, // 25 MHz + PhyIfSpeed_125M, // 125 MHz +}; + +enum TimestampSource_t : UINT32 +{ + TimestampSource_Internal = 0, + TimestampSource_External, + TimestampSource_Both, +}; + +enum ActivePhy_t : UINT32 +{ + ActivePhy_GmiiOrMii = 0, + ActivePhy_Rgmii, + ActivePhy_Sgmii, + ActivePhy_Tbi, + ActivePhy_Rmii, + ActivePhy_Rtbi, + ActivePhy_Smii, + ActivePhy_RevMii, +}; + +enum AddressWidth_t : UINT32 +{ + AddressWidth_32 = 0, + AddressWidth_40, + AddressWidth_48, +}; + +enum HashTableSize_t : UINT32 +{ + HashTableSize_0 = 0, + HashTableSize_64, + HashTableSize_128, + HashTableSize_256, +}; + +union MacConfiguration_t +{ + UINT32 Value32; + struct + { + UINT32 ReceiverEnable : 1; + UINT32 TransmitterEnable : 1; + UINT32 PreambleLength : 2; + UINT32 DeferralCheck : 1; + UINT32 BackOffLimit : 2; + UINT32 Reserved7 : 1; + + UINT32 DisableRetry : 1; + UINT32 DisableCarrierSenseDuringTransmit : 1; + UINT32 DisableReceiveOwn : 1; + UINT32 EnableCarrierSenseBeforeTransmit : 1; + UINT32 LoopbackMode : 1; + UINT32 FullDuplex : 1; + PortSelectSpeed_t PortSelectSpeed : 2; + + UINT32 JumboPacketEnable : 1; + UINT32 JabberDisable : 1; + UINT32 PacketBurstEnable : 1; + UINT32 WatchdogDisable : 1; + UINT32 PadOrCrcStripEnable : 1; + UINT32 CrcStripEnableForType : 1; + UINT32 Support2kPackets : 1; + UINT32 GiantPacketSizeLimitControlEnable : 1; + + UINT32 InterPacketGap : 3; + UINT32 ChecksumOffloadEnable : 1; + UINT32 SourceAddrInsertionControl : 3; + UINT32 ArpOffloadEnable : 1; + }; +}; + +union MacExtConfiguration_t +{ + UINT32 Value32; + struct + { + UINT32 GiantPacketSizeLimit : 14; + UINT32 Reserved14 : 2; + + UINT32 DisableCrcCheckRx : 1; + UINT32 SlowProtocolDetect : 1; + UINT32 UnicastSlowProtocolPacketDetect : 1; + UINT32 Reserved19 : 1; + UINT32 HeaderSplitMaxSize : 3; + UINT32 Reserved23 : 1; + UINT32 ExtendedIpgEnable : 1; + UINT32 ExtendedIpg : 5; + UINT32 Reserved30 : 2; + }; +}; + +union MacPacketFilter_t +{ + UINT32 Value32; + struct + { + UINT32 PromiscuousMode : 1; + UINT32 HashUnicast : 1; + UINT32 HashMulticast : 1; + UINT32 DestInverseFilter : 1; + UINT32 PassAllMulticast : 1; + UINT32 DisableBroadcast : 1; + UINT32 PassControl : 2; + + UINT32 SourceInverseFilter : 1; + UINT32 SourceFilter : 1; + UINT32 HashPerfectFilter : 1; + UINT32 Reserved11 : 5; + + UINT32 VlanTagFilter : 1; + UINT32 Reserved17 : 3; + UINT32 L3L4Filter : 1; + UINT32 DropNonTcpUdp : 1; + UINT32 Reserved22 : 9; + UINT32 ReceiveAll : 1; + }; +}; + +union MacInterruptStatus_t +{ + UINT32 Value32; + struct + { + UINT32 LinkStatus : 1; // RGSMIIIS - RGMII or SMII Interrupt Status + UINT32 Reserved1 : 2; + UINT32 Phy : 1; // PHYIS - PHY Interrupt Status + UINT32 Pmt : 1; // PMTIS - PMT Interrupt Status (magic/wake-up frame) + UINT32 Lpi : 1; // LPIIS - LPI Interrupt Status (energy efficient entry/exit) + UINT32 Reserved6 : 2; + + UINT32 Mmc : 1; // MMCIS - MMC Interrupt Status + UINT32 MmcRx : 1; // MMCRXIS - MMC Receive Interrupt Status + UINT32 MmcTx : 1; // MMCTXIS - MMC Transmit Interrupt Status + UINT32 MmcRxChecksum : 1; // MMCRXIPIS - MMC Receive Checksum Offload Interrupt Status + UINT32 Timestamp : 1; // TSIS - Timestamp Interrupt Status + UINT32 Tx : 1; // TXSTSIS - Transmit Interrupt Status + UINT32 Rx : 1; // RXSTSIS - Receive Interrupt Status + UINT32 Reserved15 : 2; + + UINT32 FramePreemption : 1; // FPEIS - Frame Preemption Interrupt Status + UINT32 Mdio : 1; // MDIOIS - MDIO Interrupt Status + UINT32 MmcFpeTx : 1; // MFTIS - MMC FPE Transmit Interrupt Status + UINT32 MmcFpeRx : 1; // MFRIS - MMC FPE Receive Interrupt Status + UINT32 Reserved21 : 11; + }; +}; + +union MacInterruptEnable_t +{ + UINT32 Value32; + struct + { + UINT32 LinkStatus : 1; // RGSMIIIE - RGMII or SMII Interrupt Enable + UINT32 Reserved1 : 2; + UINT32 Phy : 1; // PHYIE - PHY Interrupt Enable + UINT32 Pmt : 1; // PMTIE - PMT Interrupt Enable (magic/wake-up frame) + UINT32 Lpi : 1; // LPIIE - LPI Interrupt Enable (energy efficient entry/exit) + UINT32 Reserved6 : 6; + UINT32 Timestamp : 1; // TSIE - Timestamp Interrupt Enable + UINT32 Tx : 1; // TXSTSIE - Transmit Interrupt Enable + UINT32 Rx : 1; // RXSTSIE - Receive Interrupt Enable + UINT32 Reserved15 : 2; + + UINT32 FramePreemption : 1; // FPEIE - Frame Preemption Interrupt Enable + UINT32 Mdio : 1; // MDIOIE - MDIO Interrupt Enable + UINT32 Reserved19 : 13; + }; +}; + +union MacPhyIfControlStatus_t +{ + UINT32 Value32; + struct + { + UINT32 TransmitConfig : 1; // TC - Transmit Configuration to PHY + UINT32 LinkUpDuringConfig : 1; // LUD - Link Up + UINT32 Reserved2 : 14; + + UINT32 FullDuplex : 1; // LNKMODE - Link Mode + PhyIfSpeed_t Speed : 2; // SPD - Speed + UINT32 LinkUp : 1; // LNKSTS - Link Status + UINT32 Reserved20 : 12; + }; +}; + +union MacLpiControlStatus_t +{ + UINT32 Value32; + struct + { + UINT32 TransmitLpiEntry : 1; // TLPIEN - Transmit LPI Entry + UINT32 TransmitLpiExit : 1; // TLPIEX - Transmit LPI Exit + UINT32 ReceiveLpiEntry : 1; // RLPIEN - Receive LPI Entry + UINT32 ReceiveLpiExit : 1; // RLPIEX - Receive LPI Exit + UINT32 Reserved4 : 4; + + UINT32 TransmitLpiState : 1; // TLPIST - Transmit LPI State + UINT32 ReceiveLpiState : 1; // RLPIST - Receive LPI State + UINT32 Reserved10 : 6; + + UINT32 LpiEnable : 1; // LPIEN - LPI Enable + UINT32 PhyLinkStatus : 1; // PLS - PHY Link Status + UINT32 PhyLinkStatusEnable : 1; // PLSEN - PHY Link Status Enable + UINT32 LpTxAutomate : 1; // LPTXA - LPI Tx Automate + UINT32 LpiTimerEnable : 1; // LPIATE - LPI Timer Enable + UINT32 LpiTxClockStopEnable : 1; // LPITCSE - LPI Tx Clock Stop Enable + + UINT32 Reserved22 : 10; + }; +}; + +union MacVersion_t +{ + UINT32 Value32; + struct + { + UINT32 RkVer : 8; + UINT32 UserVer : 8; + UINT32 Reserved : 16; + }; +}; + +union MacHwFeature0_t +{ + UINT32 Value32; + struct + { + UINT32 Mii : 1; // MIISEL - 10 or 100 Mbps Support + UINT32 Gmii : 1; // GMIISEL - 1000 Mbps Support + UINT32 HalfDuplex : 1; // HDSEL - Half Duplex Support + UINT32 PcsSel : 1; // PCSSEL - TBI, SGMII, or RTBI PHY interface + UINT32 VlanHash : 1; // VLHASH - VLAN Hash Filter Support + UINT32 Mdio : 1; // SMASEL - MDIO Interface Support + UINT32 RemoteWake : 1; // RWKSEL - PMT Remote Wake-up Packet Enable + UINT32 MagicPacket : 1; // MGKSEL - PMT Magic Packet Enable + + UINT32 ManagementCounters : 1; // MMCSEL - MAC Management Counters + UINT32 ArpOffload : 1; // ARPOFFSEL - ARP Offload Support + UINT32 Reserved10 : 2; + UINT32 Timestamp : 1; // TSSEL - IEEE 1588-2008 Timestamp Support + UINT32 EnergyEfficient : 1; // EEESEL - Energy Efficient Ethernet Support + UINT32 TxChecksumOffload : 1; // TXCOESEL - Transmit Checksum Offload Engine + UINT32 Reserved15 : 1; + + UINT32 RxChecksumOffload : 1; // RXCOESEL - Receive Checksum Offload Engine + UINT32 Reserved17 : 1; + UINT32 MacAddrCount : 7; // ADDMACADRSEL - MAC Address Register Count + TimestampSource_t TimestampSource : 2; // TSSTSSEL - IEEE 1588-2008 Timestamp Source + UINT32 SaVlanIns : 1; // SAVLANINS - Source Address or VLAN Insertion + ActivePhy_t ActivePhy : 4; // ACTPHYSEL - Active PHY Selected + }; +}; + +union MacHwFeature1_t +{ + UINT32 Value32; + struct + { + UINT32 RxFifoSize : 5; // RXFIFOSIZE - MTL Receive FIFO Size = 128 << RxFifoSize + UINT32 SinglePortRam : 1; // SPRAM - Single Port RAM + UINT32 TxFifoSize : 5; // TXFIFOSIZE - MTL Transmit FIFO Size = 128 << RxFifoSize + UINT32 OsTen : 1; // OSTEN - One-Step Timestamping Enable + UINT32 PtpOff : 1; // PTOEN - PTP Offload Support + UINT32 AdvTHword : 1; // ADVTHWORD - IEEE 1588 High Word Register + AddressWidth_t AddressWidth : 2; // ADDR64 - Address width + + UINT32 DcbEn : 1; // DCBEN - Data Center Bridging Support + UINT32 SplitHeader : 1; // SPHEN - Split Header Support + UINT32 TsoEn : 1; // TSOEN - TCP Segmentation Offload Support + UINT32 DmaDebug : 1; // DBGMEMA - DMA Debug Support + UINT32 AvSel : 1; // AVSEL - AV Bridging Support + UINT32 RavSel : 1; // RAVSEL - Rx AV Bridging Support + UINT32 Reserved22 : 1; + UINT32 PtpOneStep : 1; // POUOST - One Step for PTP over UDP/IP Feature Enable + + HashTableSize_t HashTableSize : 2; // HASHTBLSZ - Hash Table Size + UINT32 Reserved26 : 1; + UINT32 L3L4Filters : 4; // L3L4FNUM - Number of L3/L4 Filters + UINT32 Reserved31 : 1; + }; +}; + +union MacHwFeature2_t +{ + UINT32 Value32; + struct + { + UINT32 RxQCnt : 4; // RXQCNT - Number of Rx Queues + UINT32 Reserved4 : 2; + UINT32 TxQCnt : 4; // TXQCNT - Number of Tx Queues + UINT32 Reserved10 : 2; + UINT32 RxChCnt : 4; // RXCHCNT - Number of DMA Receive Channels + + UINT32 Reserved16 : 2; + UINT32 TxChCnt : 4; // TXCHCNT - Number of DMA Transmit Channels + UINT32 Reserved22 : 2; + + UINT32 PpsOutputs : 3; // PPSOUTNUM - Number of PPS Outputs + UINT32 Reserved27 : 1; + UINT32 AuxSnapNum : 3; // AUXSNAPNUM - Number of Auxiliary Snapshot Inputs + UINT32 Reserved31 : 1; + }; +}; + +union MacHwFeature3_t +{ + UINT32 Value32; + struct + { + UINT32 ExtendedVlanTagFilters : 3; // NRVF - Number of Extended VLAN Tag Filters + UINT32 Reserved3 : 1; + UINT32 ChannelVlanTx : 1; // CBTISEL - Channel VLAN Tx Support + UINT32 DoubleVlanTag : 1; // DVLAN - Double VLAN Tag Support + UINT32 Reserved6 : 3; + UINT32 PacketDuplication : 1; // PDUPSEL - Broadcast/Multicast Packet Duplication Support + UINT32 FlexibleReceiveParser : 1; // FRPSEL - Flexible Receive Parser Support + UINT32 FlexibleReceiveBufferSize : 2; // FRPBS - Flexible Receive Parser Buffer Size + UINT32 FlexibleReceiveEntries : 2; // FRPES - Flexible Receive Parser Entries + UINT32 Reserved15 : 1; + + UINT32 EnhancedScheduling : 1; // ESTSEL - Enhancements to Scheduling Traffic Enable + UINT32 GateControlDepth : 3; // ESTDEP - Depth of the Gate Control List + UINT32 GateControlTime : 2; // ESTWID - Width of the Time Interval field in the Gate Control List + UINT32 Reserved22 : 4; + UINT32 FramePreemption : 1; // FPESEL - Frame Preemption Enable + UINT32 TimeBasedScheduling : 1; // TBSSEL - Time-Based Scheduling Enable + UINT32 AutomotiveSafety : 2; // ASP - Automotive Safety Package Enable + UINT32 Reserved31 : 2; + }; +}; + +union DmaSysBusMode_t +{ + UINT32 Value32; + struct + { + UINT32 FixedBurst : 1; // FB - Fixed Burst Length + UINT32 BurstLength4 : 1; // BLEN4 - AXI Burst Length 4 + UINT32 BurstLength8 : 1; // BLEN8 - AXI Burst Length 8 + UINT32 BurstLength16 : 1; // BLEN16 - AXI Burst Length 16 + UINT32 Reserved4 : 6; + UINT32 AutoAxiLpi : 1; // AAL - Auto AXI LPI + UINT32 Reserved11 : 1; + UINT32 AddressAlignedBeats : 1; // AAL - Address Aligned Beats + UINT32 Reserved13 : 1; + UINT32 Reserved14 : 1; // mixed-burst? + UINT32 Reserved15 : 1; + + UINT32 AxiMaxReadOutstanding : 4; // RD_OSR_LMT - AXI Maximum Read Outstanding Request Limit + UINT32 Reserved20 : 4; + UINT32 AxiMaxWriteOutstanding : 4; // WR_OSR_LMT - AXI Maximum Write Outstanding Request Limit + UINT32 Reserved28 : 2; + UINT32 UnlockOnPacket : 1; // LPI_XIT_PKT - Unlock on Magic/Remote Wake-up Packet + UINT32 EnableLpi : 1; // EN_LPI - Enable Low Power Interface (LPI) + }; +}; + +union MacTxFlowCtrl_t +{ + UINT32 Value32; + struct + { + UINT32 FlowControlBusyActivate : 1; // FCB_BPA - Flow Control Busy Activate + UINT32 TransmitFlowControlEnable : 1; // TFE - Transmit Flow Control Enable + UINT32 Reserved2 : 2; + UINT32 PauseLowThreshold : 3; // PLT - Pause Low Threshold + UINT32 DisableZeroQuantaPause : 1; // DZPQ - Disable Zero-Quanta Pause + + UINT32 Reserved8 : 8; + + UINT32 PauseTime : 16; // PT - Pause Time + }; +}; + +struct MacRegisters +{ + // MAC_Configuration @ 0x0000 = 0x0: + // The MAC Configuration Register establishes the operating mode of the MAC. + MacConfiguration_t MacConfiguration; + + // MAC_Ext_Configuration @ 0x0004 = 0x0: + // The MAC Extended Configuration Register establishes the operating mode of the + // MAC. + MacExtConfiguration_t MacExtConfiguration; + + // MAC_Packet_Filter @ 0x0008 = 0x0: + // The MAC Packet Filter register contains the filter controls for receiving + // packets. + MacPacketFilter_t MacPacketFilter; + + // MAC_Watchdog_Timeout @ 0x000C = 0x0: + // The Watchdog Timeout register controls the watchdog timeout for received + // packets. + ULONG MacWatchdogTimeout; + + // MAC_Hash_Table_RegX @ 0x0010 = 0x0: + // The Hash Table Register X contains the Xth 32 bits of the hash table. + ULONG MacHashTable[3]; + + ULONG Padding001C[13]; + + // MAC_VLAN_Tag_Ctrl @ 0x0050 = 0x0: + // This register is the redefined format of the MAC VLAN Tag Register. It is used + // for indirect addressing. It contains the address offset, command type and Busy + // Bit for CSR access of the Per VLAN Tag registers. + ULONG MacVlanTagCtrl; + + // MAC_VLAN_Tag_Data @ 0x0054 = 0x0: + // This register holds the read/write data for Indirect Access of the Per VLAN Tag + // registers.During the read access, this field contains valid read data only + // after the OB bit is reset. During the write access, this field should be valid + // prior to setting the OB bit in the MacVlanTag_Ctrl Register. + ULONG MacVlanTagData; + + // MAC_VLAN_Hash_Table @ 0x0058 = 0x0: + // When VTHM bit of the MacVlanTag register is set, the 16-bit VLAN Hash Table + // register is used for group address filtering based on the VLAN tag. + ULONG MacVlanHashTable; + + ULONG Padding005C[1]; + + // MAC_VLAN_Incl @ 0x0060 = 0x0: + // The VLAN Tag Inclusion or Replacement register contains the VLAN tag for + // insertion or replacement in the Transmit packets. It also contains the VLAN tag + // insertion controls. + ULONG MacVlanIncl; + + // MAC_Inner_VLAN_Incl @ 0x0064 = 0x0: + // The Inner VLAN Tag Inclusion or Replacement register contains the inner VLAN + // tag to be inserted or replaced in the Transmit packet. It also contains the + // inner VLAN tag insertion controls. + ULONG MacInner_VlanIncl; + + ULONG Padding0068[2]; + + // MAC_Q0_Tx_Flow_Ctrl @ 0x0070 = 0x0: + // The Flow Control register controls the generation and reception of the Control + // (Pause Command) packets by the Flow control module of the MAC. + MacTxFlowCtrl_t MacTxFlowCtrl; + + ULONG Padding0074[7]; + + // MAC_Rx_Flow_Ctrl @ 0x0090 = 0x0: + // The Receive Flow Control register controls the pausing of MAC Transmit based on + // the received Pause packet. + ULONG MacRxFlowCtrl; + + // MAC_RxQ_Ctrl4 @ 0x0094 = 0x0: + // The Receive Queue Control 4 register controls the routing of unicast and + // multicast packets that fail the Destination or Source address filter to the Rx + // queues. + ULONG MacRxCtrl4; + + ULONG Padding0098[2]; + + // MAC_RxQ_Ctrl0 @ 0x00A0 = 0x0: + // The Receive Queue Control 0 register controls the queue management in the MAC + // Receiver. + ULONG MacRxCtrl0; + + // MAC_RxQ_Ctrl1 @ 0x00A4 = 0x0: + // The Receive Queue Control 1 register controls the routing of multicast, + // broadcast, AV, DCB, and untagged packets to the Rx queues. + ULONG MacRxCtrl1; + + // MAC_RxQ_Ctrl2 @ 0x00A8 = 0x0: + // This register controls the routing of tagged packets based on the USP (user + // Priority) field of the received packets to the RxQueues 0 to 3. + ULONG MacRxCtrl2; + + ULONG Padding00AC[1]; + + // MAC_Interrupt_Status @ 0x00B0 = 0x0: + // The Interrupt Status register contains the status of interrupts. + MacInterruptStatus_t MacInterruptStatus; + + // MAC_Interrupt_Enable @ 0x00B4 = 0x0: + // The Interrupt Enable register contains the masks for generating the interrupts. + MacInterruptEnable_t MacInterruptEnable; + + // MAC_Rx_Tx_Status @ 0x00B8 = 0x0: + // The Receive Transmit Status register contains the Receive and Transmit Error + // status. + ULONG MacRxTxStatus; + + ULONG Padding00BC[1]; + + // MAC_PMT_Control_Status @ 0x00C0 = 0x0: + // The PMT Control and Status Register. + ULONG MacPmtControlStatus; + + // MAC_RWK_Packet_Filter @ 0x00C4 = 0x0: + // The Remote Wakeup Filter registers are implemented as 8, 16, or 32 indirect + // access registers (wkuppktfilter_reg#i) based on whether 4, 8, or 16 Remote + // Wakeup Filters are selected in the configuration and accessed by application + // through MacRWK_Packet_Filter register. + ULONG MacRwkPacketFilter; + + // RWK_Filter01_CRC @ 0x00C4 = 0x0: + // RWK Filter 0/1 CRC-16. + //ULONG RWK_Filter01_CRC; // UNION + + // RWK_Filter0_Byte_Mask @ 0x00C4 = 0x0: + // RWK Filter0 Byte Mask. + //ULONG RWK_Filter0_Byte_Mask; // UNION + + // RWK_Filter1_Byte_Mask @ 0x00C4 = 0x0: + // RWK Filter1 Byte Mask. + //ULONG RWK_Filter1_Byte_Mask; // UNION + + // RWK_Filter23_CRC @ 0x00C4 = 0x0: + // RWK Filter 2/3 CRC-16. + //ULONG RWK_Filter23_CRC; // UNION + + // RWK_Filter2_Byte_Mask @ 0x00C4 = 0x0: + // RWK Filter2 Byte Mask. + //ULONG RWK_Filter2_Byte_Mask; // UNION + + // RWK_Filter3_Byte_Mask @ 0x00C4 = 0x0: + // RWK Filter3 Byte Mask. + //ULONG RWK_Filter3_Byte_Mask; // UNION + + // RWK_Filter_Command @ 0x00C4 = 0x0: + // RWK Filter Command. + //ULONG RWK_Filter_Command; // UNION + + // RWK_Filter_Offset @ 0x00C4 = 0x0: + // RWK Filter Offset. + //ULONG RWK_Filter_Offset; // UNION + + ULONG Padding00C8[2]; + + // MAC_LPI_Control_Status @ 0x00D0 = 0x0: + // The LPI Control and Status Register controls the LPI functions and provides the + // LPI interrupt status. The status bits are cleared when this register is read. + MacLpiControlStatus_t MacLpiControlStatus; + + // MAC_LPI_Timers_Control @ 0x00D4 = 0x3E80000: + // The LPI Timers Control register controls the timeout values in the LPI states. + ULONG MacLpiTimersControl; + + // MAC_LPI_Entry_Timer @ 0x00D8 = 0x0: + // This register controls the Tx LPI entry timer. + ULONG MacLpiEntryTimer; + + // MAC_1US_Tic_Counter @ 0x00DC = 0x3F: + // This register controls the generation of the Reference time (1 microsecond + // tic). + ULONG Mac1usTicCounter; + + ULONG Padding00E0[6]; + + // MAC_PHYIF_Control_Status @ 0x00F8 = 0x0: + // PHY Interface Control and Status Register. + MacPhyIfControlStatus_t MacPhyIfControlStatus; + + ULONG Padding00FC[5]; + + // MAC_Version @ 0x0110 = 0x3051: + // The version register identifies the version of the GMAC. + MacVersion_t MacVersion; + + // MAC_Debug @ 0x0114 = 0x0: + // The Debug register provides the debug status of various MAC blocks. + ULONG MacDebug; + + ULONG Padding0118[1]; + + // MAC_HW_Feature0 @ 0x011C = 0x181173F3: + // This register indicates the presence of first set of the optional features or + // functions. + MacHwFeature0_t MacHwFeature0; + + // MAC_HW_Feature1 @ 0x0120 = 0x111E01E8: + // This register indicates the presence of second set of the optional features or + // functions. + MacHwFeature1_t MacHwFeature1; + + // MAC_HW_Feature2 @ 0x0124 = 0x11041041: + // This register indicates the presence of third set of the optional features or + // functions. + MacHwFeature2_t MacHwFeature2; + + // MAC_HW_Feature3 @ 0x0128 = 0xC370031: + // This register indicates the presence of fourth set the optional features or + // functions. + MacHwFeature3_t MacHwFeature3; + + ULONG Padding012C[53]; + + // MAC_MDIO_Address @ 0x0200 = 0x0: + // The MDIO Address register controls the management cycles to external PHY + // through a management interface. + ULONG MacMdioAddress; + + // MAC_MDIO_Data @ 0x0204 = 0x0: + // The MDIO Data register stores the Write data to be written to the PHY register + // located at the address specified in MacMDIO_Address. + ULONG MacMdioData; + + ULONG Padding0208[2]; + + // MAC_ARP_Address @ 0x0210 = 0x0: + // The ARP Address register contains the IPv4 Destination Address of the MAC. + ULONG MacArpAddress; + + ULONG Padding0214[7]; + + // MAC_CSR_SW_Ctrl @ 0x0230 = 0x0: + // This register contains SW programmable controls for changing the CSR access + // response and status bits clearing. + ULONG MacCsrSwCtrl; + + // MAC_FPE_CTRL_STS @ 0x0234 = 0x0: + // This register controls the operation of Frame Preemption. + ULONG MacFpeCtrlSts; + + // MAC_Ext_Cfg1 @ 0x0238 = 0x0: + // This register contains Split mode control field and offset field for Split + // Header feature. + ULONG MacExtCfg1; + + ULONG Padding023C[1]; + + // MAC_Presn_Time_ns @ 0x0240 = 0x0: + // This register contains the 32- bit binary rollover equivalent time of the PTP + // System Time in ns. + ULONG MacPresnTimeNS; + + // MAC_Presn_Time_Updt @ 0x0244 = 0x0: + // This field holds the 32-bit value of MAC 1722 Presentation Time in ns, that + // should be added to the Current Presentation Time Counter value. Init happens + // when TSINIT is set, and update happens when the TSUPDT bit is set (TSINIT and + // TSINIT defined in MacTimestamp_Control register. + ULONG MacPresnTimeUpdt; + + ULONG Padding0248[46]; + + // MAC_AddressX_High, MacAddressX_Low @ 0x0300, 0x0308, 0x0310, 0x0318. + MacAddressRegisters MacAddress[4]; + + ULONG Padding0320[248]; + + // MMC_Control @ 0x0700 = 0x0: + // This register establishes the operating mode of MMC. + ULONG MmcControl; + + // MMC_Rx_Interrupt @ 0x0704 = 0x0: + // Maintains the interrupts generated from all Receive statistics counters. + ULONG MmcRxInterrupt; + + // MMC_Tx_Interrupt @ 0x0708 = 0x0: + // Maintains the interrupts generated from all Transmit statistics counters. + ULONG MmcTxInterrupt; + + // MMC_Rx_Interrupt_Mask @ 0x070C = 0x0: + // This register maintains the masks for interrupts generated from all Receive + // statistics counters. + ULONG MmcRxInterruptMask; + + // MMC_Tx_Interrupt_Mask @ 0x0710 = 0x0: + // This register maintains the masks for interrupts generated from all Transmit + // statistics counters. + ULONG MmcTxInterruptMask; + + // Tx_Octet_Count_Good_Bad @ 0x0714 = 0x0: + // This register provides the number of bytes transmitted by the GMAC, exclusive + // of preamble and retried bytes, in good and bad packets. + ULONG TxOctetCountGoodBad; + + // Tx_Packet_Count_Good_Bad @ 0x0718 = 0x0: + // This register provides the number of good and bad packets, exclusive of retried + // packets. + ULONG TxPacketCountGoodBad; + + ULONG Padding071C[11]; + + // Tx_Underflow_Error_Packets @ 0x0748 = 0x0: + // This register provides the number of packets aborted because of packets + // underflow error. + ULONG TxUnderflowErrorPackets; + + ULONG Padding074C[5]; + + // Tx_Carrier_Error_Packets @ 0x0760 = 0x0: + // This register provides the number of packets aborted because of carrier sense + // error (no carrier or loss of carrier). + ULONG TxCarrierErrorPackets; + + // Tx_Octet_Count_Good @ 0x0764 = 0x0: + // This register provides the number of bytes exclusive of preamble, only in good + // packets. + ULONG TxOctetCountGood; + + // Tx_Packet_Count_Good @ 0x0768 = 0x0: + // This register provides the number of good packets transmitted by GMAC. + ULONG TxPacketCountGood; + + ULONG Padding076C[1]; + + // Tx_Pause_Packets @ 0x0770 = 0x0: + // This register provides the number of good Pause packets transmitted by GMAC. + ULONG TxPausePackets; + + ULONG Padding0774[3]; + + // Rx_Packets_Count_Good_Bad @ 0x0780 = 0x0: + // This register provides the number of good and bad packets received by GMAC. + ULONG RxPacketCountGoodBad; + + // Rx_Octet_Count_Good_Bad @ 0x0784 = 0x0: + // This register provides the number of bytes received by GMAC, exclusive of + // preamble, in good and bad packets. + ULONG RxOctetCountGoodBad; + + // Rx_Octet_Count_Good @ 0x0788 = 0x0: + // This register provides the number of bytes received by GMAC, exclusive of + // preamble, only in good packets. + ULONG RxOctetCountGood; + + ULONG Padding078C[1]; + + // Rx_Multicast_Packets_Good @ 0x0790 = 0x0: + // This register provides the number of good multicast packets received by GMAC. + ULONG RxMulticastPacketsGood; + + // Rx_CRC_Error_Packets @ 0x0794 = 0x0: + // This register provides the number of packets received by GMAC with CRC error. + ULONG RxCrcErrorPackets; + + ULONG Padding0798[4]; + + // Rx_Oversize_Packets_Good @ 0x07A8 = 0x0: + // This register provides the number of packets received by GMAC without errors, + // with length greater than the maxsize (1,518 bytes or 1,522 bytes for VLAN + // tagged packets; 2000 bytes if enabled in the S2KP bit of the MAC_Configuration + // register). + ULONG RxOversizePacketsGood; + + ULONG Padding07AC[7]; + + // Rx_Length_Error_Packets @ 0x07C8 = 0x0: + // This register provides the number of packets received by GMAC with length error + // (Length Type field not equal to packet size), for all packets with valid length + // field. + ULONG RxLengthErrorPackets; + + ULONG Padding07CC[1]; + + // Rx_Pause_Packets @ 0x07D0 = 0x0: + // This register provides the number of good and valid Pause packets received by + // GMAC. + ULONG RxPausePackets; + + // Rx_FIFO_Overflow_Packets @ 0x07D4 = 0x0: + // This register provides the number of missed received packets because of FIFO + // overflow. + ULONG RxFifoOverflowPackets; + + ULONG Padding07D8[1]; + + // Rx_Watchdog_Error_Packets @ 0x07DC = 0x0: + // This register provides the number of packets received by GMAC with error + // because of watchdog timeout error (packets with a data load larger than 2,048 + // bytes (when JE and WD bits are reset in MAC_Configuration register), 10,240 + // bytes (when JE bit is set and WD bit is reset in MAC_Configuration register), + // 16,384 bytes (when WD bit is set in MAC_Configuration register) or the value + // programmed in the MAC_Watchdog_Timeout register). + ULONG RxWatchdogErrorPackets; + + ULONG Padding07E0[8]; + + // MMC_IPC_Rx_Interrupt_Mask @ 0x0800 = 0x0: + // This register maintains the mask for the interrupt generated from the receive + // IPC statistic counters. + ULONG MmcIpcRxInterruptMask; + + ULONG Padding0804[1]; + + // MMC_IPC_Rx_Interrupt @ 0x0808 = 0x0: + // This register maintains the interrupt that the receive IPC statistic counters + // generate. + ULONG MmcIpcRxInterrupt; + + ULONG Padding080C[1]; + + // RxIPv4_Good_Packets @ 0x0810 = 0x0: + // This register provides the number of good IPv4 datagrams received by GMAC with + // the TCP, UDP, or ICMP payload. + ULONG RxIPv4GoodPackets; + + // RxIPv4_Header_Error_Packets @ 0x0814 = 0x0: + // This register provides the number of IPv4 datagrams received by GMAC with + // header (checksum, length, or version mismatch) errors. + ULONG RxIPv4HeaderErrorPackets; + + ULONG Padding0818[3]; + + // RxIPv6_Good_Packets @ 0x0824 = 0x0: + // This register provides the number of good IPv6 datagrams received by GMAC. + ULONG RxIPv6GoodPackets; + + // RxIPv6_Header_Error_Packets @ 0x0828 = 0x0: + // This register provides the number of IPv6 datagrams received by GMAC with + // header (length or version mismatch) errors. + ULONG RxIPv6HeaderErrorPackets; + + ULONG Padding082C[2]; + + // RxUDP_Error_Packets @ 0x0834 = 0x0: + // This register provides the number of good IP datagrams received by GMAC whose + // UDP payload has a checksum error. + ULONG RxUdpErrorPackets; + + ULONG Padding0838[1]; + + // RxTCP_Error_Packets @ 0x083C = 0x0: + // This register provides the number of good IP datagrams received by GMAC whose + // TCP payload has a checksum error. + ULONG RxTcpErrorPackets; + + ULONG Padding0840[1]; + + // RxICMP_Error_Packets @ 0x0844 = 0x0: + // This register provides the number of good IP datagrams received by GMAC whose + // ICMP payload has a checksum error. + ULONG RxIcmpErrorPackets; + + ULONG Padding0848[3]; + + // RxIPv4_Header_Error_Octets @ 0x0854 = 0x0: + // This register provides the number of bytes received by GMAC in IPv4 datagrams + // with header errors (checksum, length, version mismatch). + ULONG RxIPv4HeaderErrorOctets; + + ULONG Padding0858[4]; + + // RxIPv6_Header_Error_Octets @ 0x0868 = 0x0: + // This register provides the number of bytes received by GMAC in IPv6 datagrams + // with header errors (length, version mismatch). + ULONG RxIPv6HeaderErrorOctets; + + ULONG Padding086C[2]; + + // RxUDP_Error_Octets @ 0x0874 = 0x0: + // This register provides the number of bytes received by GMAC in a UDP segment + // that had checksum errors. + ULONG RxUdpErrorOctets; + + ULONG Padding0878[1]; + + // RxTCP_Error_Octets @ 0x087C = 0x0: + // This register provides the number of bytes received by GMAC in a TCP segment + // that had checksum errors. + ULONG RxTcpErrorOctets; + + ULONG Padding0880[1]; + + // RxICMP_Error_Octets @ 0x0884 = 0x0: + // This register provides the number of bytes received by GMAC in a good ICMP + // segment. + ULONG RxIcmpErrorOctets; + + ULONG Padding0888[6]; + + // MMC_FPE_Tx_Interrupt @ 0x08A0 = 0x0: + // This register maintains the interrupts generated from all FPE related Transmit + // statistics counters. + ULONG MmcFpeTxInterrupt; + + // MMC_FPE_Tx_Interrupt_Mask @ 0x08A4 = 0x0: + // This register maintains the masks for interrupts generated from all FPE related + // Transmit statistics counters. + ULONG MmcFpeTxInterruptMask; + + // MmcTxFpeFragment_Cntr @ 0x08A8 = 0x0: + // This register provides the number of additional mPackets transmitted due to + // preemption. + ULONG MmcTxFpeFragmentCntr; + + // MMC_Tx_Hold_Req_Cntr @ 0x08AC = 0x0: + // This register provides the count of number of times a hold request is given to + // MAC. + ULONG MmcTxHoldReqCntr; + + ULONG Padding08B0[4]; + + // MMC_FPE_Rx_Interrupt @ 0x08C0 = 0x0: + // This register maintains the interrupts generated from all FPE related Receive + // statistics counters. + ULONG MmcFpeRxInterrupt; + + // MMC_FPE_Rx_Interrupt_Mask @ 0x08C4 = 0x0: + // This register maintains the masks for interrupts generated from all FPE related + // Receive statistics counters. + ULONG MmcFpeRxInterruptMask; + + // MMC_Rx_Packet_Asm_Err_Cntr @ 0x08C8 = 0x0: + // This register provides the number of MAC frames with reassembly errors on the + // Receiver, due to mismatch in the Fragment Count value. + ULONG MmcRxPacketAsmErrCntr; + + // MMC_Rx_Packet_SMD_Err_Cntr @ 0x08CC = 0x0: + // This register provides the number of received MAC frames rejected due to + // unknown SMD value and MAC frame fragments rejected due to arriving with an + // SMD-C when there was no. + ULONG MmcRxPacketSmdErrCntr; + + // MMC_Rx_Packet_Assembly_OK_Cntr @ 0x08D0 = 0x0: + // This register provides the number of MAC frames that were successfully + // reassembled and delivered to MAC. + ULONG MmcRxPacketAssemblyOkCntr; + + // MMC_Rx_FPE_Fragment_Cntr @ 0x08D4 = 0x0: + // This register provides the number of additional mPackets transmitted due to + // preemption. + ULONG MmcRxFpeFragmentCntr; + + ULONG Padding08D8[10]; + + // MAC_L3_L4 filters @ 0x0900, 0x0930. + MacL3L4Registers MacL3L4[2]; + + ULONG Padding0950[104]; + + // MAC_Timestamp_Control @ 0x0B00 = 0x0: + // This register controls the operation of the System Time generator and + // processing of PTP packets for timestamping in the Receiver. + ULONG MacTimestampControl; + + // MAC_Sub_Second_Increment @ 0x0B04 = 0x0: + // Specifies the value to be added to the internal system time register every + // cycle of clk_ptp_ref_i clock. + ULONG MacSubSecondIncrement; + + // MAC_System_Time_Secs @ 0x0B08 = 0x0: + // The System Time Nanoseconds register, along with System Time Seconds register, + // indicates the current value of the system time maintained by the MAC. + ULONG MacSystemTimeSecs; + + // MAC_System_Time_NS @ 0x0B0C = 0x0: + // The System Time Nanoseconds register, along with System Time Seconds register, + // indicates the current value of the system time maintained by the MAC. + ULONG MacSystemTimeNS; + + // MAC_Sys_Time_Secs_Update @ 0x0B10 = 0x0: + // The System Time Seconds Update register, along with the System Time Nanoseconds + // Update register, initializes or updates the system time maintained by the MAC. + ULONG MacSysTimeSecsUpdate; + + // MAC_Sys_Time_NS_Update @ 0x0B14 = 0x0: + // MAC System Time Nanoseconds Update register. + ULONG MacSysTimeNsUpdate; + + // MAC_Timestamp_Addend @ 0x0B18 = 0x0: + // Timestamp Addend register. This register value is used only when the system + // time is configured for Fine Update mode (TSCFUPDT bit in the + // MacTimestamp_Control register). + ULONG MacTimestampAddend; + + ULONG Padding0B1C[1]; + + // MAC_Timestamp_Status @ 0x0B20 = 0x0: + // Timestamp Status register. All bits except Bits[27:25] gets cleared when the + // application reads this register. + ULONG MacTimestampStatus; + + ULONG Padding0B24[3]; + + // MAC_Tx_TS_Status_NS @ 0x0B30 = 0x0: + // This register contains the nanosecond part of timestamp captured for Transmit + // packets when Tx status is disabled. + ULONG MacTxTsStatusNS; + + // MAC_Tx_TS_Status_Secs @ 0x0B34 = 0x0: + // The register contains the higher 32 bits of the timestamp (in seconds) captured + // when a PTP packet is transmitted. + ULONG MacTxTsStatusSecs; + + ULONG Padding0B38[2]; + + // MAC_Auxiliary_Control @ 0x0B40 = 0x0: + // The Auxiliary Timestamp Control register controls the Auxiliary Timestamp + // snapshot. + ULONG MacAuxiliaryControl; + + ULONG Padding0B44[1]; + + // MAC_Auxiliary_TS_NS @ 0x0B48 = 0x0: + // The Auxiliary Timestamp Nanoseconds register, along with + // MacAuxiliary_Timestamp_Seconds, gives the 64-bit timestamp stored as auxiliary + // snapshot. + ULONG MacAuxiliaryTsNS; + + // MAC_Auxiliary_TS_Secs @ 0x0B4C = 0x0: + // The Auxiliary Timestamp - Seconds register contains the lower 32 bits of the + // Seconds field of the auxiliary timestamp register. + ULONG MacAuxiliaryTsSecs; + + ULONG Padding0B50[2]; + + // MAC_TS_Ingress_Corr_NS @ 0x0B58 = 0x0: + // This register contains the correction value in nanoseconds to be used with the + // captured timestamp value in the ingress path. + ULONG MacTsIngressCorrNS; + + // MAC_TS_Egress_Corr_NS @ 0x0B5C = 0x0: + // This register contains the correction value in nanoseconds to be used with the + // captured timestamp value in the egress path. + ULONG MacTsEgressCorrNS; + + ULONG Padding0B60[2]; + + // MAC_TS_Ingress_Latency @ 0x0B68 = 0x0: + // This register holds the Ingress MAC latency. + ULONG MacTsIngressLatency; + + // MAC_TS_Egress_Latency @ 0x0B6C = 0x0: + // This register holds the Egress MAC latency. + ULONG MacTsEgressLatency; + + // MAC_PPS_Control @ 0x0B70 = 0x0: + // PPS Control register. + ULONG MacPpsControl; + + ULONG Padding0B74[3]; + + // MAC_PPS0_Target_Time_Seconds @ 0x0B80 = 0x0: + // The PPS Target Time Seconds register, along with PPS Target Time Nanoseconds + // register, is used to schedule an interrupt event [Bit 1 of + // MacTimestamp_Status] when the system time exceeds the value programmed in + // these registers. + ULONG MacPps0TargetTimeSeconds; + + // MAC_PPS0_Target_Time_Ns @ 0x0B84 = 0x0: + // PPS0 Target Time Nanoseconds register. + ULONG MacPps0TargetTimeNs; + + // MAC_PPS0_Interval @ 0x0B88 = 0x0: + // The PPS0 Interval register contains the number of units of sub-second increment + // value between the rising edges of PPS0 signal output (ptp_pps_o[0]). + ULONG MacPps0Interval; + + // MAC_PPS0_Width @ 0x0B8C = 0x0: + // The PPS0 Width register contains the number of units of sub-second increment + // value. + ULONG MacPps0Width; + + ULONG Padding0B90[28]; + + // MTL_Operation_Mode @ 0x0C00 = 0x0: + // The Operation Mode register establishes the Transmit and Receive operating + // modes and commands. + ULONG MtlOperationMode; + + ULONG Padding0C04[1]; + + // MTL_DBG_CTL @ 0x0C08 = 0x0: + // The FIFO Debug Access Control and Status register controls the operation mode + // of FIFO debug access. + ULONG MtlDbgCtl; + + // MTL_DBG_STS @ 0x0C0C = 0x1900000: + // The FIFO Debug Status register contains the status of FIFO debug access. + ULONG MtlDdbSts; + + // MTL_FIFO_Debug_Data @ 0x0C10 = 0x0: + // The FIFO Debug Data register contains the data to be written to or read from + // the FIFOs. + ULONG MtlFifoDebugData; + + ULONG Padding0C14[3]; + + // MTL_Interrupt_Status @ 0x0C20 = 0x0: + // The software driver (application) reads this register during interrupt service + // routine or polling to determine the interrupt status of MTL queues and the MAC. + ULONG MtlInterruptStatus; + + ULONG Padding0C24[3]; + + // MTL_RxQ_DMA_Map0 @ 0x0C30 = 0x0: + // The Receive Queue and DMA Channel Mapping 0 register is reserved in EQOS-CORE + // and EQOS-MTL configurations. + ULONG MtlRxDmaMap0; + + ULONG Padding0C34[3]; + + // MTL_TBS_CTRL @ 0x0C40 = 0x0: + // This register controls the operation of Time Based Scheduling. + ULONG MtlTbsCtrl; + + ULONG Padding0C44[3]; + + // MTL_EST_Control @ 0x0C50 = 0x0: + // This register controls the operation of Enhancements to Scheduled Transmission + // (IEEE802.1Qbv). + ULONG MtlEstControl; + + ULONG Padding0C54[1]; + + // MTL_EST_Status @ 0x0C58 = 0x0: + // This register provides Status related to Enhancements to Scheduled + // Transmission. + ULONG MtlEstStatus; + + ULONG Padding0C5C[1]; + + // MTL_EST_Sch_Error @ 0x0C60 = 0x0: + // This register provides the One Hot encoded Queue Numbers that are having the + // Scheduling related error (timeout). + ULONG MtlEstSchError; + + // MTL_EST_Frm_Size_Error @ 0x0C64 = 0x0: + // This register provides the One Hot encoded Queue Numbers that are having the + // Frame. + ULONG MtlEstFrmSizeError; + + // MTL_EST_Frm_Size_Capture @ 0x0C68 = 0x0: + // This register captures the Frame Size and Queue Number of the first occurrence + // of the Frame Size related error. Up on clearing it captures the data of + // immediate next occurrence of a similar error. + ULONG MtlEstFrmSizeCapture; + + ULONG Padding0C6C[1]; + + // MTL_EST_Intr_Enable @ 0x0C70 = 0x0: + // This register implements the Interrupt Enable bits for the various events that + // generate an interrupt. Bit positions have a 1 to 1 correlation with the status + // bit positions in MtlETS_Status. + ULONG MtlEstIntrEnable; + + ULONG Padding0C74[3]; + + // MTL_EST_GCL_Control @ 0x0C80 = 0x0: + // This register provides the control information for reading/writing to the Gate + // Control lists. + ULONG MtlEstGclControl; + + // MTL_EST_GCL_Data @ 0x0C84 = 0x0: + // This register holds the read data or write data in case of reads and writes + // respectively. + ULONG MtlEstGclData; + + ULONG Padding0C88[2]; + + // MTL_FPE_CTRL_STS @ 0x0C90 = 0x0: + // This register controls the operation of, and provides status for Frame + // Preemption (IEEE802.1Qbu/802.3br). + ULONG MtlFpeCtrlSts; + + // MTL_FPE_Advance @ 0x0C94 = 0x0: + // This register holds the Hold and Release Advance time. + ULONG MtlFpeAdvance; + + ULONG Padding0C98[26]; + + // MTL_Qx @ 0x0D00, 0x0D40. + MtlQueueRegisters MtlQ[2]; + + ULONG Padding0D80[160]; + + // DMA_Mode @ 0x1000 = 0x0: + // The Bus Mode register establishes the bus operating modes for the DMA. + ULONG DmaMode; + + // DMA_SysBus_Mode @ 0x1004 = 0x10000: + // The System Bus mode register controls the behavior of the AHB or AXI master. + DmaSysBusMode_t DmaSysBusMode; + + // DMA_Interrupt_Status @ 0x1008 = 0x0: + // The application reads this Interrupt Status register during interrupt service + // routine or polling to determine the interrupt status of DMA channels, MTL + // queues, and the MAC. + ULONG DmaInterruptStatus; + + // DMA_Debug_Status0 @ 0x100C = 0x0: + // The Debug Status 0 register gives the Receive and Transmit process status for + // DMA Channel 0-Channel 2 for debugging purpose. + ULONG DmaDebugStatus0; + + ULONG Padding1010[12]; + + // AXI_LPI_Entry_Interval @ 0x1040 = 0x0: + // This register is used to control the AXI LPI entry interval. + ULONG AxiLpiEntryInterval; + + ULONG Padding1044[3]; + + // DMA_TBS_CTRL @ 0x1050 = 0x0: + // This register is used to control the TBS attributes. + ULONG DmaTbsCtrl; + + ULONG Padding1054[43]; + + // DMA_CH0 @ 0x1100. + // DMA_CH1 @ 0x1180. + ChannelRegisters DmaCh[2]; +}; +static_assert(sizeof(MacRegisters) == 0x1200); + +#pragma endregion + +#pragma region DmaDescriptor + +#define DESCRIPTOR_ALIGN 64 + +enum TxChecksumInsertion : UINT16 +{ + TxChecksumInsertionDisabled = 0, + TxChecksumInsertionEnabledHeaderOnly = 1, + TxChecksumInsertionEnabledExceptPseudo = 2, + TxChecksumInsertionEnabledIncludingPseudo = 3, +}; + +struct TxDescriptorRead +{ + // TDES0, TDES1 + + UINT32 Buf1Ap; // BUF1AP + UINT32 Buf2Ap; // BUF2AP + + // TDES2 + + UINT16 Buf1Length : 14; // B1L + UINT16 VlanTagControl : 2; // VTIR + + UINT16 Buf2Length : 14; // B2L + UINT16 TransmitTimestampEnable : 1; // TTSE + UINT16 InterruptOnCompletion : 1; // IOC + + // TDES3 + + UINT16 FrameLength; // FL (15 bits) + + TxChecksumInsertion ChecksumInsertion : 2; // CIC + UINT16 TcpSegmentationEnable : 1; // TSE = 0 + UINT16 SlotNumber : 4; // Slot Number Control Bits in AV Mode + UINT16 SourceAddressInsertionControl : 3; // SAIC + UINT16 CrcPadControl : 2; // CPC + UINT16 LastDescriptor : 1; // LD + UINT16 FirstDescriptor : 1; // FD + UINT16 ContextType : 1; // CTXT = 0 + UINT16 Own : 1; // OWN + + // DEBUG + + UINT32 FragmentIndex; + UINT32 PacketIndex; +}; +static_assert(sizeof(TxDescriptorRead) == 24); + +struct TxDescriptorReadTso +{ + // TDES0, TDES1 + + UINT32 Buf1Ap; // BUF1AP, TSO Header Address Pointer if FD = 1 + UINT32 Buf2Ap; // BUF2AP + + // TDES2 + + UINT16 Buf1Length : 14; // B1L (10-bit header length if FD = 1) + UINT16 VlanTagControl : 2; // VTIR + + UINT16 Buf2Length : 14; // B2L + UINT16 TsoMemoryWriteDisable : 1; // TMWD + UINT16 InterruptOnCompletion : 1; // IOC + + // TDES3 + + UINT32 TcpPayloadLength : 18; // TPL + UINT32 TcpSegmentationEnable : 1; // TSE = 0 + UINT32 TcpHeaderLength : 4; // TCP/UDP header length (must be 2 for UDP) + UINT32 SourceAddressInsertionControl : 3; // SAIC + UINT32 Reserved26 : 2; // CPC, ignored when TSE = 1 + UINT32 LastDescriptor : 1; // LD + UINT32 FirstDescriptor : 1; // FD + UINT32 ContextType : 1; // CTXT = 0 + UINT32 Own : 1; // OWN + + // DEBUG + + UINT32 FragmentIndex; + UINT32 PacketIndex; +}; +static_assert(sizeof(TxDescriptorReadTso) == 24); + +struct TxDescriptorWrite +{ + // TDES0, TDES1 + + UINT32 TimestampLow; // TTSL + UINT32 TimestampHigh; // TTSH + + // TDES2 + + UINT32 Reserved; + + // TDES3 + + UINT8 IpHeaderError : 1; // IHE + UINT8 DeferredBit : 1; // DB + UINT8 UnderflowError : 1; // UF + UINT8 ExcessiveDeferral : 1; // ED + UINT8 CollisionCount : 4; // CC + + UINT8 ExcessiveCollision : 1; // EC + UINT8 LateCollision : 1; // LC + UINT8 NoCarrier : 1; // NC + UINT8 LossOfCarrier : 1; // LC + UINT8 PayloadChecksumError : 1; // PCE + UINT8 PacketFlushed : 1; // PF + UINT8 JabberTimeout : 1; // JT + UINT8 ErrorSummary : 1; // ES + + UINT8 EccUncorrectableError : 1; // EUE + UINT8 TimestampStatus : 1; // TTSS + UINT8 Reserved18 : 5; + UINT8 DescriptorError : 1; // DE + + UINT8 Reserved24 : 4; + UINT8 LastDescriptor : 1; // LD + UINT8 FirstDescriptor : 1; // FD + UINT8 ContextType : 1; // CTXT = 0 + UINT8 Own : 1; // OWN + + // DEBUG + + UINT32 FragmentIndex; + UINT32 PacketIndex; +}; +static_assert(sizeof(TxDescriptorWrite) == 24); + +struct TxDescriptorContext +{ + // TDES0, TDES1 + + UINT32 TimestampLow; // TTSL + UINT32 TimestampHigh; // TTSH + + // TDES2 + + UINT16 MaximumSegmentSize : 14; // MSS + UINT16 Reserved14 : 2; + + UINT16 InnerVlanTag; // IVT + + // TDES3 + + UINT16 VlanTag; // VT + + UINT8 VlanTagValid : 1; // VLTV + UINT8 InnerVlanTagValid : 1; // IVLTV + UINT8 InnverVlanTagControl : 2; // IVTIR + UINT8 Reserved20 : 3; + UINT8 DescriptorError : 1; // DE + + UINT8 Reserved24 : 2; + UINT8 OneStepInputOrMssValid : 1; // TCMSSV + UINT8 OneStepEnable : 1; // OSTC + UINT8 Reserved28 : 2; + UINT8 ContextType : 1; // CTXT = 1 + + // DEBUG + + UINT32 FragmentIndex; + UINT32 PacketIndex; +}; +static_assert(sizeof(TxDescriptorContext) == 24); + +union TxDescriptor +{ + TxDescriptorRead Read; // When read, TSE = 0, CTXT = 0. + TxDescriptorReadTso ReadTso;// When read, TSE = 1, CTXT = 0. + TxDescriptorWrite Write; // When write, CTXT = 0. + TxDescriptorContext Context;// When CTXT = 1. + UINT32 All[DESCRIPTOR_ALIGN / sizeof(UINT32)]; +}; +static_assert(sizeof(TxDescriptor) == DESCRIPTOR_ALIGN); + +struct RxDescriptorRead +{ + // RDES0, RDES1 + + UINT32 Buf1ApLow; // BUF1AP + UINT32 Buf1ApHigh; // BUF1AP + + // RDES2 + + UINT32 Buf2Ap; // BUF2AP + + // RDES3 + + UINT16 Reserved0; + UINT8 Reserved16; + + UINT8 Buf1Valid : 1; // BUF1V + UINT8 Buf2Valid : 1; // BUF2V + UINT8 Reserved26 : 4; + UINT8 InterruptOnCompletion : 1; // IOC + UINT8 Own : 1; // OWN + + // DEBUG + + UINT32 FragmentIndex; +}; +static_assert(sizeof(RxDescriptorRead) == 20); + +struct RxDescriptorWrite +{ + // RDES0 + + UINT16 OuterVlanTag; // OVT + UINT16 InnerVlanTag; // IVT + + // RDES1 + + UINT8 PayloadType : 3; // PT + UINT8 IPHeaderError : 1; // IPHE + UINT8 IPv4HeaderPresent : 1; // IPV4 + UINT8 IPv6HeaderPresent : 1; // IPV6 + UINT8 IPChecksumBypassed : 1; // IPCB + UINT8 IPPayloadError : 1; // IPCE + + UINT8 PtpMessageType : 4; // PMT + UINT8 PtpPacketType : 1; // PFT + UINT8 PtpVersion : 1; // PV + UINT8 TimestampAvailable : 1; // TSA + UINT8 TimestampDropped : 1; // TD + + UINT16 OamSubtype; // OPC - OAM sub-type and code, or MAC control packet opcode (based on LengthType). + + // RDES2 + + UINT16 L3L4HeaderLength : 10; // HL + UINT16 ArpReplyNotGenerated : 1; // ARPNR + UINT16 Reserved11 : 3; + UINT16 InnerVlanTagStatus : 1; // ITS + UINT16 VlanFilterStatus : 1; // OTS + + UINT16 SourceFilterFail : 1; // SAF/RXPD, based on Flexible RX Parser enable. + UINT16 DestinationFilterFail : 1; // DAF/RXPI, based on Flexible RX Parser enable. + UINT16 HashFilterStatus : 1; // HF + UINT16 MacAddressMatchHashValue : 8; // MADRM + UINT16 L3FilterMatch : 1; // L3FM + UINT16 L4FilterMatch : 1; // L4FM + UINT16 L3L4FilterNumber : 3; // L3L4FM + + // RDES3 + + UINT16 PacketLength : 15; // PL + UINT16 ErrorSummary : 1; // ES + + UINT8 LengthType : 3; // LT + UINT8 DribbleBitError : 1; // DE + UINT8 ReceiveError : 1; // RE + UINT8 OverflowError : 1; // OE + UINT8 ReceiveWatchdogTimeout : 1; // RWT + UINT8 GiantPacket : 1; // GP + + UINT8 CrcError : 1; // CE + UINT8 Rdes0Valid : 1; // RS0V, valid only if LD = 1. + UINT8 Rdes1Valid : 1; // RS1V, valid only if LD = 1. + UINT8 Rdes2Valid : 1; // RS2V, valid only if LD = 1. + UINT8 LastDescriptor : 1; // LD + UINT8 FirstDescriptor : 1; // FD + UINT8 ContextType : 1; // CTXT = 0 + UINT8 Own : 1; // OWN + + // DEBUG + + UINT32 FragmentIndex; +}; +static_assert(sizeof(RxDescriptorWrite) == 20); + +struct RxDescriptorContext +{ + UINT32 TimestampLow; // RSTL + UINT32 TimestampHigh; // RSTH + UINT32 Reserved2; + + // RDES3 + + UINT16 Reserved3a; + UINT8 Reserved3b; + UINT8 Reserved3c : 5; + UINT8 DescriptorError : 1; // DE + UINT8 ContextType : 1; // CTXT = 0 + UINT8 Own : 1; // OWN + + // DEBUG + + UINT32 FragmentIndex; +}; +static_assert(sizeof(RxDescriptorContext) == 20); + +union RxDescriptor +{ + RxDescriptorRead Read; // When read. + RxDescriptorWrite Write; // When write, CTXT = 0. + RxDescriptorContext Context;// When write, CTXT = 1. + UINT32 All[DESCRIPTOR_ALIGN / sizeof(UINT32)]; +}; +static_assert(sizeof(TxDescriptor) == DESCRIPTOR_ALIGN); + +#pragma endregion + +#pragma warning(pop) diff --git a/drivers/net/dwc_eqos/rxqueue.cpp b/drivers/net/dwc_eqos/rxqueue.cpp new file mode 100644 index 0000000..23d5c5c --- /dev/null +++ b/drivers/net/dwc_eqos/rxqueue.cpp @@ -0,0 +1,379 @@ +#include "precomp.h" +#include "rxqueue.h" +#include "queue_common.h" +#include "device.h" +#include "registers.h" +#include "trace.h" + +static_assert(sizeof(RxDescriptor) == QueueDescriptorSize); + +struct RxQueueContext +{ + ChannelRegisters* channelRegs; + + NETADAPTER adapter; + NET_RING* packetRing; + NET_RING* fragmentRing; + WDFCOMMONBUFFER descBuffer; + RxDescriptor* descVirtual; + PHYSICAL_ADDRESS descPhysical; + NET_EXTENSION fragmentLogical; + UINT32 descCount; // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount. + bool running; + + UINT32 descBegin; + UINT32 descEnd; +}; +WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(RxQueueContext, RxQueueGetContext) + +// Gets channelRegs->RxCurrentAppDesc, converted to an index. +static UINT32 +GetDescNext(_In_ RxQueueContext* context) +{ + // DISPATCH_LEVEL + auto const current = Read32(&context->channelRegs->RxCurrentAppDesc); + return QueueDescriptorAddressToIndex(current, context->descPhysical, context->descCount); +} + +// Sets channelRegs->RxDescTailPointer to the physical address of the descriptor at the given index. +static void +SetDescEnd( + _In_ RxQueueContext* context, + _In_ UINT32 index) +{ + // DISPATCH_LEVEL + NT_ASSERT(index < context->descCount); + UINT32 const offset = index * sizeof(RxDescriptor); + Write32(&context->channelRegs->RxDescTailPointer, context->descPhysical.LowPart + offset); + context->descEnd = index; +} + +static EVT_PACKET_QUEUE_START RxQueueStart; +static void +RxQueueStart(_In_ NETPACKETQUEUE queue) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = RxQueueGetContext(queue); + + context->running = true; + context->descBegin = 0; + context->descEnd = 0; + + Write32(&context->channelRegs->RxDescListAddressHigh, context->descPhysical.HighPart); + Write32(&context->channelRegs->RxDescListAddress, context->descPhysical.LowPart); + Write32(&context->channelRegs->RxDescRingLength, context->descCount - 1); + SetDescEnd(context, context->descEnd); + +#if DBG + auto const descNext = GetDescNext(context); + NT_ASSERT(descNext == context->descEnd); // If this fires, we need more reset logic. +#endif + + ChannelRxControl_t rxControl = {}; + rxControl.Start = true; + rxControl.ReceiveBufferSize = RxBufferSize; + rxControl.RxPbl = QueueBurstLength; + Write32(&context->channelRegs->RxControl, rxControl); + + TraceEntryExit(RxQueueStart, LEVEL_INFO); +} + +static EVT_PACKET_QUEUE_ADVANCE RxQueueAdvance; +static void +RxQueueAdvance(_In_ NETPACKETQUEUE queue) +{ + // DISPATCH_LEVEL + auto const context = RxQueueGetContext(queue); + auto const pktEnd = context->packetRing->EndIndex; + auto const fragEnd = context->fragmentRing->EndIndex; + auto const descMask = context->descCount - 1u; + UINT32 descIndex, pktIndex, fragIndex; + UINT32 donePkts = 0, queuedPkts = 0; + + /* + Fragment indexes: + [fragBegin] old [fragNext] new [fragEnd] owned by NetAdapterCx [fragBegin] + + Descriptor indexes: + [descBegin] RECEIVED [descNext] RECEIVING [descEnd] EMPTY [descBegin-1] + */ + + // Indicate received packets. + + pktIndex = context->packetRing->BeginIndex; + fragIndex = context->fragmentRing->BeginIndex; + + auto const descNext = GetDescNext(context); + for (descIndex = context->descBegin; descIndex != descNext; descIndex = (descIndex + 1) & descMask) + { + NT_ASSERT(fragIndex != fragEnd); + if (pktIndex == pktEnd || fragIndex == fragEnd) + { + break; + } + + auto const& desc = context->descVirtual[descIndex]; + auto const descWrite = desc.Write; + + NT_ASSERT(!descWrite.Own); + if (descWrite.Own) + { + // I've never seen this happen, but if it does, we need to stop here. + TraceWrite("RxQueueAdvance-own", LEVEL_WARNING, + TraceLoggingUInt32(descIndex, "descIndex"), + TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "RDES3")); + break; + } + + NT_ASSERT(descWrite.FragmentIndex == fragIndex); + + auto const pkt = NetRingGetPacketAtIndex(context->packetRing, pktIndex); + pkt->FragmentIndex = fragIndex; + pkt->FragmentCount = 1; + pkt->Layout = {}; + + auto const frag = NetRingGetFragmentAtIndex(context->fragmentRing, fragIndex); + frag->Offset = 0; + NT_ASSERT(RxBufferSize <= frag->Capacity); + + if (descWrite.ErrorSummary || + descWrite.ContextType || + !descWrite.FirstDescriptor || + !descWrite.LastDescriptor) // TODO: Jumbo frames (multi-descriptor packets) + { + if (descWrite.ErrorSummary) + { + TraceWrite("RxQueueAdvance-Dropped", LEVEL_INFO, + TraceLoggingUInt32(descIndex, "descIndex"), + TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "RDES3")); + } + else + { + TraceWrite("RxQueueAdvance-Unexpected", LEVEL_WARNING, + TraceLoggingUInt32(descIndex, "descIndex"), + TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "RDES3")); + } + + pkt->Ignore = true; + frag->ValidLength = 0; + } + else + { + NT_ASSERT(descWrite.PacketLength >= 4); // PacketLength includes CRC + frag->ValidLength = descWrite.PacketLength - 4; + } + + pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); + fragIndex = NetRingIncrementIndex(context->fragmentRing, fragIndex); + donePkts += 1; + } + + context->descBegin = descIndex; + context->packetRing->BeginIndex = pktIndex; + context->fragmentRing->BeginIndex = fragIndex; + + if (context->running) + { + // Prepare more descriptors. + + fragIndex = context->fragmentRing->NextIndex; + + auto const descFull = (context->descBegin - 1u) & descMask; + for (descIndex = context->descEnd; descIndex != descFull; descIndex = (descIndex + 1) & descMask) + { + if (fragIndex == fragEnd) + { + break; + } + + NT_ASSERT(RxBufferSize <= NetRingGetFragmentAtIndex(context->fragmentRing, fragIndex)->Capacity); + auto const fragLogicalAddress = NetExtensionGetFragmentLogicalAddress(&context->fragmentLogical, fragIndex)->LogicalAddress; + + RxDescriptorRead descRead = {}; + descRead.Buf1ApLow = fragLogicalAddress & 0xFFFFFFFF; + descRead.Buf1ApHigh = fragLogicalAddress >> 32; + descRead.Buf1Valid = true; + descRead.InterruptOnCompletion = true; + descRead.Own = true; + descRead.FragmentIndex = fragIndex; + + context->descVirtual[descIndex].Read = descRead; + + fragIndex = NetRingIncrementIndex(context->fragmentRing, fragIndex); + queuedPkts += 1; + } + + if (descIndex != context->descEnd) + { + SetDescEnd(context, descIndex); + context->fragmentRing->NextIndex = fragIndex; + } + } + else if (descIndex == descNext) + { + while (pktIndex != pktEnd) + { + auto const pkt = NetRingGetPacketAtIndex(context->packetRing, pktIndex); + pkt->Ignore = true; + pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); + } + + context->packetRing->BeginIndex = pktEnd; + context->fragmentRing->BeginIndex = fragEnd; + } + + TraceEntryExit(RxQueueAdvance, LEVEL_VERBOSE, + TraceLoggingUInt32(donePkts), + TraceLoggingUInt32(queuedPkts)); +} + +static EVT_PACKET_QUEUE_SET_NOTIFICATION_ENABLED RxQueueSetNotificationEnabled; +static void +RxQueueSetNotificationEnabled( + _In_ NETPACKETQUEUE queue, + _In_ BOOLEAN notificationEnabled) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = RxQueueGetContext(queue); + DeviceSetNotificationRxQueue(context->adapter, notificationEnabled ? queue : nullptr); + TraceEntryExit(RxQueueSetNotificationEnabled, LEVEL_VERBOSE, + TraceLoggingBoolean(notificationEnabled, "enabled")); +} + +static EVT_PACKET_QUEUE_CANCEL RxQueueCancel; +static void +RxQueueCancel(_In_ NETPACKETQUEUE queue) +{ + // DISPATCH_LEVEL (verifier says so) + auto const context = RxQueueGetContext(queue); + + // Shut down and flush. RxQueueAdvance should return the remaining packets. + + context->running = false; + + ChannelRxControl_t rxControl = {}; + rxControl.Start = false; + rxControl.RxPacketFlush = true; + Write32(&context->channelRegs->RxControl, rxControl); + + TraceEntryExit(RxQueueCancel, LEVEL_INFO); +} + +static EVT_PACKET_QUEUE_STOP RxQueueStop; +__declspec(code_seg("PAGE")) +static void +RxQueueStop(_In_ NETPACKETQUEUE queue) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + auto const context = RxQueueGetContext(queue); + + DeviceSetNotificationRxQueue(context->adapter, nullptr); + + TraceEntryExit(RxQueueStop, LEVEL_INFO); +} + +static EVT_WDF_OBJECT_CONTEXT_CLEANUP RxQueueCleanup; +static void +RxQueueCleanup(_In_ WDFOBJECT queue) +{ + // DISPATCH_LEVEL + auto context = RxQueueGetContext(queue); + if (context->descBuffer) + { + WdfObjectDelete(context->descBuffer); + context->descBuffer = nullptr; + } + + TraceEntryExit(RxQueueCleanup, LEVEL_VERBOSE); +} + +_Use_decl_annotations_ NTSTATUS +RxQueueCreate( + NETADAPTER adapter, + NETRXQUEUE_INIT* queueInit, + WDFDMAENABLER dma, + ChannelRegisters* channelRegs) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + NTSTATUS status; + NETPACKETQUEUE queue = nullptr; + + // Create queue. + + { + NET_PACKET_QUEUE_CONFIG config; + NET_PACKET_QUEUE_CONFIG_INIT(&config, RxQueueAdvance, RxQueueSetNotificationEnabled, RxQueueCancel); + config.EvtStart = RxQueueStart; + config.EvtStop = RxQueueStop; + + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, RxQueueContext); + attributes.EvtCleanupCallback = RxQueueCleanup; + + status = NetRxQueueCreate(queueInit, &attributes, &config, &queue); + if (!NT_SUCCESS(status)) + { + TraceWrite("NetRxQueueCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + } + + // Configure queue. + + { + auto const rings = NetRxQueueGetRingCollection(queue); + + auto const context = RxQueueGetContext(queue); + context->channelRegs = channelRegs; + + context->adapter = adapter; + context->packetRing = NetRingCollectionGetPacketRing(rings); + context->fragmentRing = NetRingCollectionGetFragmentRing(rings); + context->descCount = QueueDescriptorCount(context->fragmentRing->NumberOfElements); + + TraceWrite("RxQueueCreate-size", LEVEL_VERBOSE, + TraceLoggingHexInt32(context->packetRing->NumberOfElements, "packets"), + TraceLoggingHexInt32(context->fragmentRing->NumberOfElements, "fragments"), + TraceLoggingHexInt32(context->descCount, "descriptors")); + + WDF_COMMON_BUFFER_CONFIG bufferConfig; + WDF_COMMON_BUFFER_CONFIG_INIT(&bufferConfig, QueueDescriptorAlignment - 1); + status = WdfCommonBufferCreateWithConfig( + dma, + sizeof(RxDescriptor) * context->descCount, + &bufferConfig, + WDF_NO_OBJECT_ATTRIBUTES, + &context->descBuffer); + if (!NT_SUCCESS(status)) + { + TraceWrite("WdfCommonBufferCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + + context->descVirtual = static_cast( + WdfCommonBufferGetAlignedVirtualAddress(context->descBuffer)); + memset(context->descVirtual, 0, sizeof(RxDescriptor) * context->descCount); + context->descPhysical = + WdfCommonBufferGetAlignedLogicalAddress(context->descBuffer); + TraceWrite("RxQueueCreate-desc", LEVEL_VERBOSE, + TraceLoggingHexInt64(context->descPhysical.QuadPart, "physical"), + TraceLoggingPointer(context->descVirtual, "virtual")); + + NET_EXTENSION_QUERY query; + NET_EXTENSION_QUERY_INIT(&query, + NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_NAME, + NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_VERSION_1, + NetExtensionTypeFragment); + NetRxQueueGetExtension(queue, &query, &context->fragmentLogical); + } + + status = STATUS_SUCCESS; + +Done: + + TraceEntryExitWithStatus(RxQueueCreate, LEVEL_INFO, status); + return status; +} diff --git a/drivers/net/dwc_eqos/rxqueue.h b/drivers/net/dwc_eqos/rxqueue.h new file mode 100644 index 0000000..88484dd --- /dev/null +++ b/drivers/net/dwc_eqos/rxqueue.h @@ -0,0 +1,14 @@ +#pragma once + +struct ChannelRegisters; +auto constexpr RxBufferSize = 2048u; + +_IRQL_requires_same_ +_IRQL_requires_(PASSIVE_LEVEL) +//__declspec(code_seg("PAGE")) // Nonpaged - on resume path. +NTSTATUS +RxQueueCreate( + _In_ NETADAPTER adapter, + _Inout_ NETRXQUEUE_INIT* queueInit, + _In_ WDFDMAENABLER dma, + _Inout_ ChannelRegisters* channelRegs); diff --git a/drivers/net/dwc_eqos/trace.h b/drivers/net/dwc_eqos/trace.h new file mode 100644 index 0000000..ef02741 --- /dev/null +++ b/drivers/net/dwc_eqos/trace.h @@ -0,0 +1,48 @@ +#pragma once + +// *dwc_eqos = {5d8331d3-70b3-5620-5664-db28f48a4b79} +TRACELOGGING_DECLARE_PROVIDER(TraceProvider); + +enum : UINT8 +{ + LEVEL_CRITICAL = WINEVENT_LEVEL_CRITICAL, + LEVEL_ERROR = WINEVENT_LEVEL_ERROR, + LEVEL_WARNING = WINEVENT_LEVEL_WARNING, + LEVEL_INFO = WINEVENT_LEVEL_INFO, + LEVEL_VERBOSE = WINEVENT_LEVEL_VERBOSE, +}; + +#define KW_GENERAL TraceLoggingKeyword(0x1) // All events should have at least one keyword. +#define KW_ENTRY_EXIT TraceLoggingKeyword(0x2) + +// TraceLoggingWrite(eventName, KW_GENERAL, ...). +#define TraceWrite(eventName, level, ...) \ + TraceLoggingWrite(TraceProvider, eventName, TraceLoggingLevel(level), KW_GENERAL, __VA_ARGS__) + +// TraceLoggingWrite("funcName", KW_ENTRY_EXIT, ...). +#define TraceEntryExit(funcName, level, ...) \ + TraceWrite(#funcName, level, KW_ENTRY_EXIT, __VA_ARGS__) \ + +// TraceLoggingWrite("funcName", KW_ENTRY_EXIT, status, ...) +#define TraceEntryExitWithStatus(funcName, successLevel, status, ...) \ + if (NTSTATUS const _status = (status); NT_SUCCESS(_status)) { \ + TraceEntryExit(funcName, successLevel, TraceLoggingNTStatus(_status, "status"), __VA_ARGS__); \ + } else { \ + TraceEntryExit(funcName, LEVEL_ERROR, TraceLoggingNTStatus(_status, "status"), __VA_ARGS__); \ + } \ + +// TraceLoggingWrite("funcName-Entry", KW_ENTRY_EXIT, ...). +#define TraceEntry(funcName, level, ...) \ + TraceWrite(#funcName "-Entry", level, KW_ENTRY_EXIT, __VA_ARGS__) \ + +// TraceLoggingWrite("funcName-Exit", KW_ENTRY_EXIT, ...). +#define TraceExit(funcName, level, ...) \ + TraceWrite(#funcName "-Exit", level, KW_ENTRY_EXIT, __VA_ARGS__) \ + +// TraceLoggingWrite("funcName-Exit", KW_ENTRY_EXIT, status, ...) +#define TraceExitWithStatus(funcName, successLevel, status, ...) \ + if (NTSTATUS const _status = (status); NT_SUCCESS(_status)) { \ + TraceExit(funcName, successLevel, TraceLoggingNTStatus(_status, "status"), __VA_ARGS__); \ + } else { \ + TraceExit(funcName, LEVEL_ERROR, TraceLoggingNTStatus(_status, "status"), __VA_ARGS__); \ + } \ diff --git a/drivers/net/dwc_eqos/txqueue.cpp b/drivers/net/dwc_eqos/txqueue.cpp new file mode 100644 index 0000000..3ae4b00 --- /dev/null +++ b/drivers/net/dwc_eqos/txqueue.cpp @@ -0,0 +1,480 @@ +#include "precomp.h" +#include "txqueue.h" +#include "queue_common.h" +#include "device.h" +#include "registers.h" +#include "trace.h" + +static_assert(sizeof(TxDescriptor) == QueueDescriptorSize); + +struct TxQueueContext +{ + ChannelRegisters* channelRegs; + MtlQueueRegisters* mtlRegs; + NETADAPTER adapter; + NET_RING* packetRing; + NET_RING* fragmentRing; + WDFCOMMONBUFFER descBuffer; + TxDescriptor* descVirtual; + PHYSICAL_ADDRESS descPhysical; + NET_EXTENSION fragmentLogical; + UINT32 descCount; // A power of 2 between QueueDescriptorMinCount and QueueDescriptorMaxCount. + + UINT32 descBegin; // Start of the TRANSMIT region. + UINT32 descEnd; // End of the TRANSMIT region, start of the EMPTY region. +}; +WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(TxQueueContext, TxQueueGetContext) + +// Gets channelRegs->TxCurrentAppDesc, converted to an index. +static UINT32 +GetDescNext(_In_ TxQueueContext* context) +{ + // DISPATCH_LEVEL + auto const current = Read32(&context->channelRegs->TxCurrentAppDesc); + return QueueDescriptorAddressToIndex(current, context->descPhysical, context->descCount); +} + +// Sets channelRegs->TxDescTailPointer to the physical address of the descriptor at the given index. +static void +SetDescEnd( + _In_ TxQueueContext* context, + _In_ UINT32 index) +{ + // DISPATCH_LEVEL + NT_ASSERT(index < context->descCount); + UINT32 const offset = index * sizeof(TxDescriptor); + Write32(&context->channelRegs->TxDescTailPointer, context->descPhysical.LowPart + offset); + context->descEnd = index; +} + +static EVT_PACKET_QUEUE_START TxQueueStart; +static void +TxQueueStart(_In_ NETPACKETQUEUE queue) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = TxQueueGetContext(queue); + + context->descBegin = 0; + context->descEnd = 0; + + Write32(&context->channelRegs->TxDescListAddressHigh,context->descPhysical.HighPart); + Write32(&context->channelRegs->TxDescListAddress, context->descPhysical.LowPart); + Write32(&context->channelRegs->TxDescRingLength, context->descCount - 1); + SetDescEnd(context, context->descEnd); + +#if DBG + auto const descNext = GetDescNext(context); + NT_ASSERT(descNext == context->descEnd); // If this fires, we need more reset logic. +#endif + + ChannelTxControl_t txControl = {}; + txControl.Start = true; + txControl.OperateOnSecondPacket = true; + txControl.TxPbl = QueueBurstLength; + Write32(&context->channelRegs->TxControl, txControl); + + TraceEntryExit(TxQueueStart, LEVEL_INFO); +} + +static EVT_PACKET_QUEUE_ADVANCE TxQueueAdvance; +static void +TxQueueAdvance(_In_ NETPACKETQUEUE queue) +{ + // DISPATCH_LEVEL + auto const context = TxQueueGetContext(queue); + auto const pktBegin = context->packetRing->BeginIndex; + auto const pktNext = context->packetRing->NextIndex; + auto const pktEnd = context->packetRing->EndIndex; + auto const descMask = context->descCount - 1u; + UINT32 descIndex, pktIndex, fragIndex; + UINT32 donePkts = 0, queuedPkts = 0, queuedFrags = 0; + + /* + Packet indexes: + [pktBegin] old [pktNext] new [pktEnd] owned by NetAdapterCx [pktBegin] + + Descriptor indexes: + [descBegin] TRANSMITTED [descNext] TRANSMITTING [descEnd] EMPTY [descBegin-1] + */ + + // Indicate transmitted packets. + + pktIndex = pktBegin; + fragIndex = context->fragmentRing->BeginIndex; + + auto const descNext = GetDescNext(context); + descIndex = context->descBegin; + auto descReady = (descNext - descIndex) & descMask; // Number of descriptors ready to be indicated. + while (descIndex != descNext) + { + NT_ASSERT(pktIndex != pktNext); + + auto const pkt = NetRingGetPacketAtIndex(context->packetRing, pktIndex); + auto const fragmentCount = pkt->FragmentCount; + if (!pkt->Ignore) + { + NT_ASSERT(fragmentCount != 0); + + if (fragmentCount > descReady) + { + break; + } + + for (unsigned i = 0; i != fragmentCount; i += 1) + { + auto const descIndex2 = (descIndex + i) & descMask; + NT_ASSERT(descIndex2 != descNext); + + auto const& desc = context->descVirtual[descIndex2]; + auto const descWrite = desc.Write; + NT_ASSERT(descWrite.PacketIndex == pktIndex); + NT_ASSERT(descWrite.FragmentIndex == NetRingAdvanceIndex(context->fragmentRing, pkt->FragmentIndex, i)); + + // Descriptor is still owned by the DMA engine? + if (descWrite.Own) + { + /* + There's a race condition where we see the update to TxCurrentAppDesc + before we see the update to the descriptor. It seems harmless (we + only use the descriptor for assertions and logging), and a fix for + this would likely affect performance, so just stop when we see a + descriptor with the Own bit still set. + + This would be a bigger issue if it happened in the Rx path, but + I've never seen that happen. + */ + TraceWrite("TxQueueAdvance-own", LEVEL_VERBOSE, + TraceLoggingUInt32(descIndex2, "descIndex"), + TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "TDES3"), + TraceLoggingUInt32(i, "fragment"), + TraceLoggingUInt32(fragmentCount)); + goto DoneIndicating; + } + else if ( + descWrite.ErrorSummary || + descWrite.ContextType || + descWrite.DescriptorError || + (descWrite.LastDescriptor != 0) != (i == fragmentCount - 1u)) + { + TraceWrite("TxQueueAdvance-error", LEVEL_ERROR, + TraceLoggingUInt32(descIndex2, "descIndex"), + TraceLoggingHexInt32(reinterpret_cast(&descWrite)[3], "TDES3"), + TraceLoggingUInt32(i, "fragment"), + TraceLoggingUInt32(fragmentCount)); + } + } + + donePkts += 1; + descReady -= fragmentCount; + descIndex = (descIndex + fragmentCount) & descMask; + } + + pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); + fragIndex = NetRingAdvanceIndex(context->fragmentRing, pkt->FragmentIndex, fragmentCount); + } + +DoneIndicating: + + context->packetRing->BeginIndex = pktIndex; + context->fragmentRing->BeginIndex = fragIndex; + context->descBegin = descIndex; + + // Fill more descriptors. + + pktIndex = pktNext; + + // Number of EMPTY is (descBegin-1) - descEnd, wrapping around if necessary. + auto const descEnd = context->descEnd; + auto descEmpty = ((context->descBegin - 1) - descEnd) & descMask; + descIndex = descEnd; + while (descEmpty != 0) + { + NT_ASSERT(descIndex != ((context->descBegin - 1) & descMask)); + + if (pktIndex == pktEnd) + { + break; + } + + auto const pkt = NetRingGetPacketAtIndex(context->packetRing, pktIndex); + if (!pkt->Ignore) + { + fragIndex = pkt->FragmentIndex; + + auto const fragmentCount = pkt->FragmentCount; + NT_ASSERT(fragmentCount != 0); + if (fragmentCount > descEmpty) + { + break; + } + + UINT32 frameLength = 0; + for (unsigned i = 0, fragIndex2 = fragIndex; i != fragmentCount; i += 1) + { + frameLength += NetRingGetFragmentAtIndex(context->fragmentRing, fragIndex2)->ValidLength & 0x03FFFFFF; // 26 bits + fragIndex2 = NetRingIncrementIndex(context->fragmentRing, fragIndex2); + } + NT_ASSERT(frameLength <= 0x7FFF); + frameLength &= 0x7FFF; + + for (unsigned i = 0; i != fragmentCount; i += 1) + { + auto const frag = NetRingGetFragmentAtIndex(context->fragmentRing, fragIndex); + NT_ASSERT(frag->ValidLength <= frag->Capacity - frag->Offset); + auto const fragLogicalAddress = NetExtensionGetFragmentLogicalAddress(&context->fragmentLogical, fragIndex)->LogicalAddress; + + TxDescriptorRead descRead = {}; + descRead.Buf1Ap = static_cast(fragLogicalAddress); + descRead.Buf2Ap = static_cast(fragLogicalAddress >> 32); + NT_ASSERT(frag->ValidLength <= 0x3FFF); // 14 bits + descRead.Buf1Length = frag->ValidLength & 0x3FFF; + descRead.InterruptOnCompletion = true; + descRead.FrameLength = static_cast(frameLength); + descRead.LastDescriptor = i == fragmentCount - 1u; + descRead.FirstDescriptor = i == 0; + descRead.Own = true; + descRead.PacketIndex = pktIndex; + descRead.FragmentIndex = fragIndex; + + context->descVirtual[descIndex].Read = descRead; + descIndex = (descIndex + 1) & descMask; + fragIndex = NetRingIncrementIndex(context->fragmentRing, fragIndex); + queuedFrags += 1; + } + + queuedPkts += 1; + } + + pktIndex = NetRingIncrementIndex(context->packetRing, pktIndex); + } + + // In some error cases, the device may stall until we write to the tail pointer + // again. Write to the tail pointer if there are pending descriptors, even if we + // didn't fill any new ones. + if (descIndex != descNext) + { + SetDescEnd(context, descIndex); + context->packetRing->NextIndex = pktIndex; + } + + TraceEntryExit(TxQueueAdvance, LEVEL_VERBOSE, + TraceLoggingUInt32(donePkts), + TraceLoggingUInt32(queuedPkts), + TraceLoggingUInt32(queuedFrags)); +} + +static EVT_PACKET_QUEUE_SET_NOTIFICATION_ENABLED TxQueueSetNotificationEnabled; +static void +TxQueueSetNotificationEnabled( + _In_ NETPACKETQUEUE queue, + _In_ BOOLEAN notificationEnabled) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + auto const context = TxQueueGetContext(queue); + DeviceSetNotificationTxQueue(context->adapter, notificationEnabled ? queue : nullptr); + TraceEntryExit(TxQueueSetNotificationEnabled, LEVEL_VERBOSE, + TraceLoggingBoolean(notificationEnabled, "enabled")); +} + +static EVT_PACKET_QUEUE_CANCEL TxQueueCancel; +static void +TxQueueCancel(_In_ NETPACKETQUEUE queue) +{ + // DISPATCH_LEVEL (verifier says so) + auto const context = TxQueueGetContext(queue); + unsigned char mode; // For tracing: 0 = empty, 1 = flushed, 2 = reset. + unsigned retry; + + // Shut down. + + ChannelTxControl_t txControl = {}; + txControl.Start = false; + Write32(&context->channelRegs->TxControl, txControl); + + // Flush. + + if (context->packetRing->BeginIndex == context->packetRing->EndIndex) + { + // Queue empty. + mode = 0; + retry = 0; + NT_ASSERT(Read32(&context->mtlRegs->TxDebug).NotEmpty == 0); + } + else + { + // Wait for a potential in-progress packet to be sent. + + for (retry = 0; retry != 10; retry += 1) + { + auto debug = Read32(&context->mtlRegs->TxDebug); + + if (debug.ReadStatus != MtlTxReadStatus_Read && debug.NotEmpty == 0) + { + break; + } + + KeStallExecutionProcessor(20); + } + + if (retry != 0) + { + // Last chance for any packets to be reported as sent. + mode = 1; + TxQueueAdvance(queue); + } + else + { + // Give up and reset the Tx queue. + mode = 2; + TraceWrite("TxQueueCancel-timeout", LEVEL_WARNING); + auto txOperationMode = Read32(&context->mtlRegs->TxOperationMode); + txOperationMode.FlushTxQueue = true; + Write32(&context->mtlRegs->TxOperationMode, txOperationMode); + for (retry = 0;; retry += 1) + { + KeStallExecutionProcessor(20); + if (!Read32(&context->mtlRegs->TxOperationMode).FlushTxQueue) + { + break; + } + } + + context->descBegin = 0; + context->descEnd = 0; + } + + NT_ASSERT(context->descBegin == context->descEnd); + NT_ASSERT(context->descBegin == GetDescNext(context)); + + // Discard any packets that are still in the queue. + + context->packetRing->BeginIndex = context->packetRing->EndIndex; + context->fragmentRing->BeginIndex = context->fragmentRing->EndIndex; + } + + TraceEntryExit(TxQueueCancel, LEVEL_INFO, + TraceLoggingUInt8(mode), // 0 = empty, 1 = flushed, 2 = reset. + TraceLoggingUInt32(retry)); +} + +static EVT_PACKET_QUEUE_STOP TxQueueStop; +__declspec(code_seg("PAGE")) +static void +TxQueueStop(_In_ NETPACKETQUEUE queue) +{ + // PASSIVE_LEVEL + PAGED_CODE(); + auto const context = TxQueueGetContext(queue); + + context->descBegin = 0; + context->descEnd = 0; + DeviceSetNotificationTxQueue(context->adapter, nullptr); + + TraceEntryExit(TxQueueStop, LEVEL_INFO); +} + +static EVT_WDF_OBJECT_CONTEXT_CLEANUP TxQueueCleanup; +static void +TxQueueCleanup(_In_ WDFOBJECT queue) +{ + // DISPATCH_LEVEL + auto context = TxQueueGetContext(queue); + if (context->descBuffer) + { + WdfObjectDelete(context->descBuffer); + context->descBuffer = nullptr; + } + + TraceEntryExit(TxQueueCleanup, LEVEL_VERBOSE); +} + +_Use_decl_annotations_ NTSTATUS +TxQueueCreate( + NETADAPTER adapter, + NETTXQUEUE_INIT* queueInit, + WDFDMAENABLER dma, + ChannelRegisters* channelRegs, + MtlQueueRegisters* mtlRegs) +{ + // PASSIVE_LEVEL, nonpaged (resume path) + NTSTATUS status; + NETPACKETQUEUE queue = nullptr; + + // Create queue. + + { + NET_PACKET_QUEUE_CONFIG config; + NET_PACKET_QUEUE_CONFIG_INIT(&config, TxQueueAdvance, TxQueueSetNotificationEnabled, TxQueueCancel); + config.EvtStart = TxQueueStart; + config.EvtStop = TxQueueStop; + + WDF_OBJECT_ATTRIBUTES attributes; + WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, TxQueueContext); + attributes.EvtCleanupCallback = TxQueueCleanup; + + status = NetTxQueueCreate(queueInit, &attributes, &config, &queue); + if (!NT_SUCCESS(status)) + { + TraceWrite("NetTxQueueCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + } + + // Configure queue. + + { + auto const rings = NetTxQueueGetRingCollection(queue); + + auto const context = TxQueueGetContext(queue); + context->channelRegs = channelRegs; + context->mtlRegs = mtlRegs; + context->adapter = adapter; + context->packetRing = NetRingCollectionGetPacketRing(rings); + context->fragmentRing = NetRingCollectionGetFragmentRing(rings); + context->descCount = QueueDescriptorCount(context->fragmentRing->NumberOfElements); + + TraceWrite("TxQueueCreate-size", LEVEL_VERBOSE, + TraceLoggingHexInt32(context->packetRing->NumberOfElements, "packets"), + TraceLoggingHexInt32(context->fragmentRing->NumberOfElements, "fragments"), + TraceLoggingHexInt32(context->descCount, "descriptors")); + + WDF_COMMON_BUFFER_CONFIG bufferConfig; + WDF_COMMON_BUFFER_CONFIG_INIT(&bufferConfig, QueueDescriptorAlignment - 1); + status = WdfCommonBufferCreateWithConfig( + dma, + sizeof(TxDescriptor) * context->descCount, + &bufferConfig, + WDF_NO_OBJECT_ATTRIBUTES, + &context->descBuffer); + if (!NT_SUCCESS(status)) + { + TraceWrite("WdfCommonBufferCreate-failed", LEVEL_ERROR, + TraceLoggingNTStatus(status)); + goto Done; + } + + context->descVirtual = static_cast( + WdfCommonBufferGetAlignedVirtualAddress(context->descBuffer)); + memset(context->descVirtual, 0, sizeof(TxDescriptor) * context->descCount); + context->descPhysical = + WdfCommonBufferGetAlignedLogicalAddress(context->descBuffer); + TraceWrite("TxQueueCreate-desc", LEVEL_VERBOSE, + TraceLoggingHexInt64(context->descPhysical.QuadPart, "physical"), + TraceLoggingPointer(context->descVirtual, "virtual")); + + NET_EXTENSION_QUERY query; + NET_EXTENSION_QUERY_INIT(&query, + NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_NAME, + NET_FRAGMENT_EXTENSION_LOGICAL_ADDRESS_VERSION_1, + NetExtensionTypeFragment); + NetTxQueueGetExtension(queue, &query, &context->fragmentLogical); + } + + status = STATUS_SUCCESS; + +Done: + + TraceEntryExitWithStatus(TxQueueCreate, LEVEL_INFO, status); + return status; +} diff --git a/drivers/net/dwc_eqos/txqueue.h b/drivers/net/dwc_eqos/txqueue.h new file mode 100644 index 0000000..c29289b --- /dev/null +++ b/drivers/net/dwc_eqos/txqueue.h @@ -0,0 +1,15 @@ +#pragma once + +struct ChannelRegisters; +struct MtlQueueRegisters; + +_IRQL_requires_same_ +_IRQL_requires_(PASSIVE_LEVEL) +//__declspec(code_seg("PAGE")) // Nonpaged - on resume path. +NTSTATUS +TxQueueCreate( + _In_ NETADAPTER adapter, + _Inout_ NETTXQUEUE_INIT* queueInit, + _In_ WDFDMAENABLER dma, + _Inout_ ChannelRegisters* channelRegs, + _Inout_ MtlQueueRegisters* mtlRegs);