diff --git a/holoviews/plotting/bokeh/annotation.py b/holoviews/plotting/bokeh/annotation.py
index 201d397d24..1bef23a4c1 100644
--- a/holoviews/plotting/bokeh/annotation.py
+++ b/holoviews/plotting/bokeh/annotation.py
@@ -1,3 +1,4 @@
+import itertools
 from collections import defaultdict
 from html import escape
 
@@ -51,7 +52,11 @@ def get_data(self, element, ranges, style):
 
     def initialize_plot(self, ranges=None, plot=None, plots=None, source=None):
         figure = super().initialize_plot(ranges=ranges, plot=plot, plots=plots, source=source)
-        labels = "yx" if self.invert_axes else "xy"
+        # Only force labels if no other ranges are set
+        if self.overlaid and set(itertools.chain.from_iterable(ranges)) - {"HSpans", "VSpans", "VLines", "HLines"}:
+            return figure
+        labels = [self.xlabel or "x", self.ylabel or "y"]
+        labels = labels[::-1] if self.invert_axes else labels
         for ax, label in zip(figure.axis, labels):
             ax.axis_label = label
         return figure
diff --git a/holoviews/tests/plotting/bokeh/test_annotationplot.py b/holoviews/tests/plotting/bokeh/test_annotationplot.py
index b9da8a1139..3525e80a1d 100644
--- a/holoviews/tests/plotting/bokeh/test_annotationplot.py
+++ b/holoviews/tests/plotting/bokeh/test_annotationplot.py
@@ -232,6 +232,15 @@ def test_hlines_plot(self):
         assert (source.data["y"] == [0, 1, 2, 5.5]).all()
         assert (source.data["extra"] == [-1, -2, -3, -44]).all()
 
+    def test_hlines_xlabel_ylabel(self):
+        hlines = HLines(
+            {"y": [0, 1, 2, 5.5], "extra": [-1, -2, -3, -44]}, vdims=["extra"]
+        ).opts(xlabel="xlabel", ylabel="xlabel")
+        plot = bokeh_renderer.get_plot(hlines)
+        assert isinstance(plot.handles["glyph"], BkHSpan)
+        assert plot.handles["xaxis"].axis_label == "xlabel"
+        assert plot.handles["yaxis"].axis_label == "xlabel"
+
     def test_hlines_array(self):
         hlines = HLines(np.array([0, 1, 2, 5.5]))
         plot = bokeh_renderer.get_plot(hlines)
@@ -357,6 +366,18 @@ def test_vlines_hlines_overlay(self):
         assert plot.handles["y_range"].start == 0
         assert plot.handles["y_range"].end == 5.5
 
+    def test_vlines_hlines_overlay_non_annotation(self):
+        non_annotation = hv.Curve([], kdims=["time"])
+        hlines = HLines(
+            {"y": [0, 1, 2, 5.5], "extra": [-1, -2, -3, -44]}, vdims=["extra"]
+        )
+        vlines = VLines(
+            {"x": [0, 1, 2, 5.5], "extra": [-1, -2, -3, -44]}, vdims=["extra"]
+        )
+        plot = bokeh_renderer.get_plot(non_annotation * hlines * vlines)
+        assert plot.handles["xaxis"].axis_label == "time"
+        assert plot.handles["yaxis"].axis_label == "y"
+
     def test_coloring_hline(self):
         hlines = HLines({"y": [1, 2, 3]})
         hlines = hlines.opts(
@@ -400,6 +421,15 @@ def test_hspans_plot(self):
         assert (source.data["y1"] == [1, 4, 6.5]).all()
         assert (source.data["extra"] == [-1, -2, -3]).all()
 
+    def test_hspans_plot_xlabel_ylabel(self):
+        hspans = HSpans(
+            {"y0": [0, 3, 5.5], "y1": [1, 4, 6.5], "extra": [-1, -2, -3]}, vdims=["extra"]
+        ).opts(xlabel="xlabel", ylabel="xlabel")
+        plot = bokeh_renderer.get_plot(hspans)
+        assert isinstance(plot.handles["glyph"], BkHStrip)
+        assert plot.handles["xaxis"].axis_label == "xlabel"
+        assert plot.handles["yaxis"].axis_label == "xlabel"
+
     def test_hspans_plot_invert_axes(self):
         hspans = HSpans(
             {"y0": [0, 3, 5.5], "y1": [1, 4, 6.5], "extra": [-1, -2, -3]}, vdims=["extra"]
@@ -514,6 +544,18 @@ def test_vspans_hspans_overlay(self):
         assert plot.handles["y_range"].start == 0
         assert plot.handles["y_range"].end == 6.5
 
+    def test_vlines_hlines_overlay_non_annotation(self):
+        non_annotation = hv.Curve([], kdims=["time"])
+        hspans = HSpans(
+            {"y0": [0, 3, 5.5], "y1": [1, 4, 6.5], "extra": [-1, -2, -3]}, vdims=["extra"]
+        )
+        vspans = VSpans(
+            {"x0": [0, 3, 5.5], "x1": [1, 4, 6.5], "extra": [-1, -2, -3]}, vdims=["extra"]
+        )
+        plot = bokeh_renderer.get_plot(non_annotation * hspans * vspans)
+        assert plot.handles["xaxis"].axis_label == "time"
+        assert plot.handles["yaxis"].axis_label == "y"
+
     def test_coloring_hline(self):
         hspans = HSpans({"y0": [1, 3, 5], "y1": [2, 4, 6]}).opts(
             alpha=hv.dim("y0").norm(),