Repeated calls to Overlay, Curve, and .opts() causes StoreOptions to fail

[dycw]

ALL software version info

Python 3.9, with a simple Makefile:

.ONESHELL:
SHELL=/bin/bash
.PHONY: env
env:
        [ -d .venv ] && rm -rf .venv/
        PYENV_VERSION=3.9.13 python -m venv --clear .venv
        ! [ -L activate ] && ln -s .venv/bin/activate
        source activate
        pip install --upgrade numpy holoviews hypothesis pip pre-commit pytest

Run with make env.

Description of expected behavior and the observed behavior

Not to crash. Instead, hypothesis marks this as a flaky test: one that fails initially, but then succeeded upon shrinking.

Complete, minimal, self-contained example code that reproduces the issue

# test_plots.py

from holoviews import Curve, Overlay
from holoviews.plotting import bokeh  # type: ignore # noqa
from hypothesis import given, settings
from hypothesis.extra.numpy import arrays
from hypothesis.strategies import data, integers, lists


@given(data=data())
@settings(deadline=None, max_examples=10000)
def test_main(data):
    n = data.draw(integers(1, 5))
    arrs = data.draw(lists(arrays(float, n), min_size=1, max_size=5))

    _ = Overlay(items=[Curve(arr) for arr in arrs])  # this will succeed

    if True:  # this will fail
        _ = Overlay(items=[Curve(arr).opts(show_grid=True) for arr in arrs])

    if False:  # this will fail
        _ = Overlay(items=[Curve(arr).opts(tools=["hover"]) for arr in arrs])

Stack traceback and/or browser JavaScript console output

Run with pytest test_plots.py:

E           hypothesis.errors.Flaky: Hypothesis test_main(data=data(...)) produces unreliable results: Falsified on the first call but did not on a subsequent one
E           Falsifying example: test_main(
E               data=data(...),
E           )
E           Draw 1: 3
E           Draw 2: [array([6.10351562e-05, 6.10351562e-05, 6.10351562e-05])]
E           Failed to reproduce exception. Expected:
E           data = data(...)
E
E               @given(data=data())
E               @settings(max_examples=1000)
E               def test_main(data):
E                   n = data.draw(integers(1, 5))
E                   arrs = data.draw(
E                       lists(
E                           arrays(float, n, elements=floats(-1.0, 1.0)),
E                           min_size=1,
E                           max_size=5,
E                       )
E                   )
E                   if False:  # this will succeed
E                       _ = Overlay(items=[Curve(arr).opts() for arr in arrs])
E
E                   if False:  # this will succeed
E                       _ = Overlay(items=[Curve(arr).opts(show_grid=True) for arr in arrs])
E
E                   if True:  # this will fail
E           >           _ = Overlay(items=[Curve(arr).opts(tools=["hover"]) for arr in arrs])
E
E           test_plots.py:26:
E           _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
E           test_plots.py:26: in <listcomp>
E               _ = Overlay(items=[Curve(arr).opts(tools=["hover"]) for arr in arrs])
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:45: in pipelined_call
E               result = __call__(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:571: in __call__
E               return self._dispatch_opts( *args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:575: in _dispatch_opts
E               return self._base_opts(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:654: in _base_opts
E               return self._obj.options(*new_args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/data/__init__.py:204: in pipelined_fn
E               result = method_fn(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/data/__init__.py:1212: in options
E               return super(Dataset, self).options(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/dimension.py:1283: in options
E               obj = obj.opts._dispatch_opts(expanded, backend=backend, clone=clone)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:575: in _dispatch_opts
E               return self._base_opts(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:651: in _base_opts
E               return opts.apply_groups(self._obj, **dict(kwargs, **new_kwargs))
E           .venv/lib/python3.9/site-packages/holoviews/util/__init__.py:225: in apply_groups
E               obj = cls._apply_groups_to_backend(obj, backend_opts, backend, clone)
E           .venv/lib/python3.9/site-packages/holoviews/util/__init__.py:150: in _apply_groups_to_backend
E               return StoreOptions.set_options(obj_handle, options, backend=backend)
E           .venv/lib/python3.9/site-packages/holoviews/core/options.py:1858: in set_options
E               custom_trees, id_mapping = cls.create_custom_trees(obj, spec, backend=backend)
E           .venv/lib/python3.9/site-packages/holoviews/core/options.py:1648: in create_custom_trees
E               offset = cls.id_offset()
E           _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
E
E           cls = <class 'holoviews.core.options.StoreOptions'>
E
E               @classmethod
E               def id_offset(cls):
E                   """
E                   Compute an appropriate offset for future id values given the set
E                   of ids currently defined across backends.
E                   """
E                   max_ids = []
E                   for backend in Store.renderers.keys():
E                       store_ids = Store.custom_options(backend=backend).keys()
E           >           max_id = max(store_ids)+1 if len(store_ids) > 0 else 0
E           E           ValueError: max() arg is an empty sequence
E
E           .venv/lib/python3.9/site-packages/holoviews/core/options.py:1783: ValueError

[hoxbro]

This seems to happen because of a race condition happening between len(store_ids) and max(store_ids). It could be solved by converting store_ids to a list in the above line. Thoughts @jlstevens?

holoviews/holoviews/core/options.py

Lines 1782 to 1783 in 6c0dafb

	store_ids = Store.custom_options(backend=backend).keys()
	max_id = max(store_ids)+1 if len(store_ids) > 0 else 0

I'm curious about why you need to run this test with so many iterations.

[dycw]

Hi @hoxbro , hypothesis defaults to 100 examples, but for this MCRE, I raised the example count higher so it fails with near certainty.

[maximlt]

This allows to almost always reproduce the error. I've seen it at the iteration 3, 4, and 5.

import numpy as np
import holoviews as hv

from holoviews import Curve, Overlay

hv.extension('bokeh')


arrs = [np.random.rand(10) for _ in range(5)]

for i in range(1000):
    try:
        Overlay(items=[Curve(arr).opts(show_grid=True) for arr in arrs])
    except ValueError:
        raise RuntimeError(f'Found it at iteration {i-1}')

[dycw]

Thank you @maximlt .

[github-actions]

This issue has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.