diff --git a/Sources/NIOCore/ChannelHandlers.swift b/Sources/NIOCore/ChannelHandlers.swift
index b63afc12dd..599071fdf2 100644
--- a/Sources/NIOCore/ChannelHandlers.swift
+++ b/Sources/NIOCore/ChannelHandlers.swift
@@ -68,8 +68,9 @@ public final class AcceptBackoffHandler: ChannelDuplexHandler, RemovableChannelH
                     scheduleRead(at: self.nextReadDeadlineNS!, context: context)
                 }
             }
+        } else {
+            context.fireErrorCaught(error)
         }
-        context.fireErrorCaught(error)
     }
 
     public func channelInactive(context: ChannelHandlerContext) {
diff --git a/Sources/NIOPosix/Bootstrap.swift b/Sources/NIOPosix/Bootstrap.swift
index c7c1f18dd4..c0a06c8f2e 100644
--- a/Sources/NIOPosix/Bootstrap.swift
+++ b/Sources/NIOPosix/Bootstrap.swift
@@ -648,6 +648,10 @@ extension ServerBootstrap {
                 serverChannelInit(serverChannel)
             }.flatMap { (_) -> EventLoopFuture<NIOAsyncChannel<ChannelInitializerResult, Never>> in
                 do {
+                    try serverChannel.pipeline.syncOperations.addHandler(
+                        AcceptBackoffHandler(),
+                        name: "AcceptBackOffHandler"
+                    )
                     try serverChannel.pipeline.syncOperations.addHandler(
                         AcceptHandler(childChannelInitializer: childChannelInit, childChannelOptions: childChannelOptions),
                         name: "AcceptHandler"