Only force labels if type no other Element are used

holoviews/plotting/bokeh/annotation.py

@@ -1,3 +1,4 @@
+import itertools
 from collections import defaultdict
 from html import escape
 
@@ -52,7 +53,8 @@ def get_data(self, element, ranges, style):
 
     def initialize_plot(self, ranges=None, plot=None, plots=None, source=None):
         figure = super().initialize_plot(ranges=ranges, plot=plot, plots=plots, source=source)
-        if self.overlaid:
+        # Only force labels if no other ranges are set
+        if self.overlaid and set(itertools.chain.from_iterable(ranges)) - {"HSpans", "VSpans", "VLines", "HLines"}:
             return figure
         labels = "yx" if self.invert_axes else "xy"
         for ax, label in zip(figure.axis, labels):