From 84f5cdd658e3e8362ef5a38958ac3a4b3055e763 Mon Sep 17 00:00:00 2001 From: Alex Rousskov Date: Tue, 17 Sep 2024 22:04:21 +0000 Subject: [PATCH] Fix ENTRY_ABORTED assertion in sendClientOldEntry() (#1903) FATAL: assertion failed: client_side_reply.cc:392: "!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)" The replaced assertion was wrong because a stale entry may be aborted while we are revalidating it. The exact real-world conditions that triggered this assertion are unknown, but many conditions lead to aborted entries. The assertion can be triggered in lab tests using a hit transaction that collapses on a miss transaction that runs into storage problems (e.g., a memory cache that ran out of usable space). Also adjusted cacheHit() to avoid similar problems. We have not reproduced them, but no code maintains the asserted invariant on the cacheHit() path either. Moreover, async-called cacheHit() initiates processExpired() that leads to problematic sendClientOldEntry() call, so seeing an aborted StoreEntry at cacheHit() time is probably a matter of sufficient concurrency levels and asynchronous callback delays. --- src/client_side_reply.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/client_side_reply.cc b/src/client_side_reply.cc index 3deab499dc8..db7d80d78c5 100644 --- a/src/client_side_reply.cc +++ b/src/client_side_reply.cc @@ -388,8 +388,15 @@ clientReplyContext::sendClientOldEntry() { /* Get the old request back */ restoreState(); + + if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) { + debugs(88, 3, "stale entry aborted while we revalidated: " << *http->storeEntry()); + http->updateLoggingTags(LOG_TCP_MISS); + processMiss(); + return; + } + /* here the data to send is in the next nodes buffers already */ - assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)); Assure(matchesStreamBodyBuffer(lastStreamBufferedBytes)); Assure(!lastStreamBufferedBytes.offset); sendMoreData(lastStreamBufferedBytes); @@ -551,7 +558,12 @@ clientReplyContext::cacheHit(const StoreIOBuffer result) return; } - assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); + if (EBIT_TEST(e->flags, ENTRY_ABORTED)) { + debugs(88, 3, "refusing aborted " << *e); + http->updateLoggingTags(LOG_TCP_MISS); + processMiss(); + return; + } /* * Got the headers, now grok them