Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
Log the details of background update failures (#16212)
Browse files Browse the repository at this point in the history
  • Loading branch information
David Robertson authored Sep 1, 2023
1 parent ed5e8a7 commit 6525fd6
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 3 deletions.
1 change: 1 addition & 0 deletions changelog.d/16212.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Log the details of background update failures.
4 changes: 2 additions & 2 deletions synapse/storage/background_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,14 +405,14 @@ async def run_background_updates(self, sleep: bool) -> None:
try:
result = await self.do_next_background_update(sleep)
back_to_back_failures = 0
except Exception:
except Exception as e:
logger.exception("Error doing update: %s", e)
back_to_back_failures += 1
if back_to_back_failures >= 5:
self._aborted = True
raise RuntimeError(
"5 back-to-back background update failures; aborting."
)
logger.exception("Error doing update")
else:
if result:
logger.info(
Expand Down
24 changes: 23 additions & 1 deletion tests/storage/test_background_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from unittest.mock import AsyncMock, Mock

import yaml
Expand Down Expand Up @@ -330,6 +330,28 @@ async def update_short(progress: JsonDict, count: int) -> int:
self.update_handler.side_effect = update_short
self.get_success(self.updates.do_next_background_update(False))

def test_failed_update_logs_exception_details(self) -> None:
needle = "RUH ROH RAGGY"

def failing_update(progress: JsonDict, count: int) -> int:
raise Exception(needle)

self.update_handler.side_effect = failing_update
self.update_handler.reset_mock()

self.get_success(
self.store.db_pool.simple_insert(
"background_updates",
values={"update_name": "test_update", "progress_json": "{}"},
)
)

with self.assertLogs(level=logging.ERROR) as logs:
# Expect a back-to-back RuntimeError to be raised
self.get_failure(self.updates.run_background_updates(False), RuntimeError)

self.assertTrue(any(needle in log for log in logs.output), logs.output)


class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase):
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
Expand Down

0 comments on commit 6525fd6

Please sign in to comment.