Updates to backlight management and standalone/hosted mode transitions (

#3)

* Cleaner handling of standalone/hosted mode state cache

* Clean up transitions between hosted and standalone modes

* Fix event expectations in standalone tests

* Disable backlight management by default

It causes issues with LEDs, and probably is a bit intrusive.

* Send a backlight update on disconnect if configured

* Force-update LEDs a few seconds after setting backlight

Works around a firmware issue where the LEDs revert to their values
from one of the standalone presets after sending a backlight sysex.

* Fix type check error

* Remove references to SSCOM port name

CONTRIBUTING.md

@@ -9,8 +9,8 @@ that source won't show up until tests are actually running, so you'll
 need to configure the control surface manually while you're running
 tests for the first time.
 
-You can add the test control surface in addition to the main SSCOM
-control surface, if you don't want to deal with switching the
+You can safely add this test control surface alongside your primary
+modeStep control surface, if you don't want to deal with switching the
 input/output when you want to run tests.
 
 To run tests, use:
@@ -25,13 +25,19 @@ For debug output:
 DEBUG=1 make test
 ```
 
+To run only specs tagged with `@now`:
+
+```shell
+.venv/bin/pytest -m now
+```
+
 ## Linting and type checks
 
 Before submitting a PR, make sure the following are passing:
 
 ```shell
-make lint # Validates code style
-make check # Validates types
+make lint # Validates code style.
+make check # Validates types.
 ```
 
 Some lint errors can be fixed automatically with:

Makefile

@@ -1,5 +1,8 @@
 SYSTEM_MIDI_REMOTE_SCRIPTS_DIR := /Applications/Ableton\ Live\ 12\ Suite.app/Contents/App-Resources/MIDI\ Remote\ Scripts
-TEST_PROJECT_DIR = tests/modeStep_tests_project
+
+TEST_PROJECT_SET_NAMES := backlight default overrides standalone wide_clip_launch
+TEST_PROJECT_DIR := tests/modeStep_tests_project
+TEST_PROJECT_SETS := $(addprefix $(TEST_PROJECT_DIR)/, $(addsuffix .als, $(TEST_PROJECT_SET_NAMES)))
 
 .PHONY: deps
 deps: __ext__/System_MIDIRemoteScripts/.make.decompile .make.pip-install
@@ -19,7 +22,7 @@ check: .make.pip-install __ext__/System_MIDIRemoteScripts/.make.decompile
 	.venv/bin/pyright .
 
 .PHONY: test
-test: .make.pip-install $(TEST_PROJECT_DIR)/default.als $(TEST_PROJECT_DIR)/overrides.als $(TEST_PROJECT_DIR)/standalone.als $(TEST_PROJECT_DIR)/wide_clip_launch.als
+test: .make.pip-install $(TEST_PROJECT_SETS)
 	.venv/bin/pytest
 
 .PHONY: img

control_surface/__init__.py

@@ -2,6 +2,7 @@
 
 import logging
 import typing
+from contextlib import contextmanager
 from functools import partial
 
 from ableton.v3.base import const, depends, inject, listens, task
@@ -44,7 +45,13 @@
 from .session import SessionComponent
 from .session_navigation import SessionNavigationComponent
 from .session_ring import SessionRingComponent
-from .sysex import DEVICE_FAMILY_BYTES, MANUFACTURER_ID_BYTES
+from .sysex import (
+    DEVICE_FAMILY_BYTES,
+    MANUFACTURER_ID_BYTES,
+    SYSEX_BACKLIGHT_OFF_REQUEST,
+    SYSEX_BACKLIGHT_ON_REQUEST,
+    SYSEX_STANDALONE_MODE_ON_REQUESTS,
+)
 from .track_controls import TrackControlsComponent, TrackControlsState
 from .transport import TransportComponent
 from .types import Action, TrackControl
@@ -176,6 +183,14 @@ class Specification(ControlSurfaceSpecification):
         *DEVICE_FAMILY_BYTES,
     )
 
+    # Force the controller into standalone mode when exiting (this will be redundant if
+    # a standalone mode is already active.) The disconnect program change message will
+    # be appended below, if configured.
+    goodbye_messages: typing.Collection[
+        typing.Tuple[int, ...]
+    ] = SYSEX_STANDALONE_MODE_ON_REQUESTS
+    send_goodbye_messages_last = True
+
     component_map = {
         "Clip_Actions": ClipActionsComponent,
         "Device": create_device_component,
@@ -217,12 +232,29 @@ def __init__(self, specification=Specification, *a, c_instance=None, **k):
         specification.link_session_ring_to_track_selection = (
             self._configuration.link_session_ring_to_track_selection
         )
-
-        super().__init__(*a, specification=specification, c_instance=c_instance, **k)
+        if self._configuration.disconnect_program is not None:
+            specification.goodbye_messages = [
+                *specification.goodbye_messages,
+                (0xC0, self._configuration.disconnect_program),
+            ]
+        if self._configuration.disconnect_backlight is not None:
+            specification.goodbye_messages = [
+                *specification.goodbye_messages,
+                (
+                    SYSEX_BACKLIGHT_ON_REQUEST
+                    if self._configuration.disconnect_backlight
+                    else SYSEX_BACKLIGHT_OFF_REQUEST
+                ),
+            ]
 
         # Internal tracker during connect/reconnect events.
         self._mode_after_identified = self._configuration.initial_mode
 
+        # For hacking around the weird LED behavior when updating the backlight.
+        self.__is_suppressing_hardware: bool = False
+
+        super().__init__(*a, specification=specification, c_instance=c_instance, **k)
+
     # Dependencies to be injected throughout the application.
     #
     # We need the `Any` return type because otherwise the type checker
@@ -250,38 +282,29 @@ def _create_elements(self, specification: ControlSurfaceSpecification):  # type:
             return super(modeStep, modeStep)._create_elements(specification)
 
     def setup(self):
-        # Activate the background mode before doing anything. No-op if no background
-        # program has been set.
-        self.component_map[
-            "Hardware"
-        ].standalone_program = self._configuration.background_program
-        self._flush_midi_messages()
-
-        # Put the controller explicitly into hosted mode. This avoids
-        # the need to modify this attribute in every non-standalone
-        # control surface mode, which would send unnecessary sysex
-        # messages on every mode change.
-        self.component_map["Hardware"].standalone = False
-
         super().setup()
 
-        logger.info(f"{self.__class__.__name__} setup complete")
+        hardware = self.component_map["Hardware"]
+        # Activate the background program before doing anything. The program change will
+        # get sent when the controller is placed into `_stanadlone_init_mode`. No-op if
+        # no background program has been set.
+        hardware.standalone_program = self._configuration.background_program
 
-    def disconnect(self):
-        # The individual control element `disconnect()` methods will
-        # be called, which should clear all lights and put the
-        # controller into standalone mode. The order of MIDI events is
-        # not guaranteed, but it shouldn't really matter, since LED
-        # updates will be applied to the current standalone preset
-        # regardless of whether the controller is in standalone or
-        # hosted mode.
-        super().disconnect()
-
-        # Send the final program change message, if any.
-        if self._configuration.disconnect_program:
-            self._send_midi(
-                (0xC0, self._configuration.disconnect_program), optimized=False
-            )
+        # Turn on hosted mode by default, so it doesn't need to be specified explicitly
+        # in normal (non-standalone) mode layers.
+        hardware.standalone = False
+
+        # Activate `_disabled` mode, which will enable the hardware controller in its
+        # `on_leave` callback.
+        self.main_modes.selected_mode = DISABLED_MODE_NAME
+
+        # Listen for backlight color values, to hack around the weird LED behavior when
+        # the backlight sysexes get sent.
+        assert self.__on_backlight_send_value is not None
+        assert self.elements
+        self.__on_backlight_send_value.subject = self.elements.backlight_sysex
+
+        logger.info(f"{self.__class__.__name__} setup complete")
 
     @property
     def main_modes(self):
@@ -301,15 +324,17 @@ def on_identified(self, response_bytes):
         if not self._identity_response_timeout_task.is_killed:
             self._identity_response_timeout_task.kill()
 
-        # Don't do anything unless we're currently in disabled
-        # mode. There's no need to force a controller update.
+        # We'll reach this point on startup, as well as when MIDI ports change (due to
+        # hardware disconnects/reconnects or changes in the Live settings). Don't do
+        # anything unless we're currently in disabled mode, i.e. unless we're
+        # transitioning from a disconnected controller - there's no need to switch in
+        # and out of standalone mode otherwise.
         if (
             self.main_modes.selected_mode is None
             or self.main_modes.selected_mode == DISABLED_MODE_NAME
         ):
-            # Force the controller into standalone mode (so we're starting
-            # from a consistent state), send the standalone background
-            # program if any, and clear the LEDs and display.
+            # Force the controller into standalone mode, and send the standalone
+            # background program, if any.
             self.main_modes.selected_mode = STANDALONE_INIT_MODE_NAME
 
             # After a short delay, load the main desired mode. This
@@ -335,13 +360,13 @@ def _on_identity_response_timeout(self):
         ):
             self._mode_after_identified = self.main_modes.selected_mode
 
-        # Enter disabled mode, which relinquishes control of
-        # everything. This ensures that nothing will be bound when the
-        # controller is identified, so we won't send a bunch of LED
-        # messages before placing it into hosted mode. (This could
-        # still happen if the controller were connected and
-        # disconnected quickly, but in any case it would just
-        # potentially mess with standalone-mode LEDs.)
+        # Enter disabled mode, which relinquishes control of everything. This ensures
+        # that sysex state values will be invalidated (by disconnecting their control
+        # elements), and nothing will be bound when the controller is next identified,
+        # so we won't send a bunch of LED messages before placing it into hosted
+        # mode. (This could still happen if the controller were connected and
+        # disconnected quickly, but in any case it would just potentially mess with
+        # standalone-mode LEDs.)
         self.main_modes.selected_mode = DISABLED_MODE_NAME
 
     @listens("is_identified")
@@ -384,3 +409,62 @@ def _after_identified(self):
             else self._configuration.initial_mode
         )
         self.main_modes.selected_mode = mode
+
+    # Whenever a backlight sysex is fired, after several seconds, the LEDs revert to the
+    # initial colors of the most recent standalone preset (even when in hosted
+    # mode). This appears to be a firmware bug, as the behavior is also reproducible
+    # when setting the backlight via the SoftStep editor. Work around this by refreshing
+    # device state a few times after the appropriate wait.
+    @lazy_attribute
+    def _backlight_workaround_task(self):
+        def refresh_state_except_backlight():
+            with self.__suppressing_backlight():
+                # Clears all send caches and updates all components.
+                self.update()
+
+        backlight_workaround_task = self._tasks.add(
+            task.sequence(
+                task.wait(3.5),
+                # Keep trying for a bit, sometimes the LEDs blank out later than
+                # expected.
+                *[
+                    task.sequence(
+                        task.run(refresh_state_except_backlight), task.wait(0.2)
+                    )
+                    for _ in range(10)
+                ],
+            )
+        )
+        backlight_workaround_task.kill()
+        return backlight_workaround_task
+
+    @listens("send_value")
+    def __on_backlight_send_value(self, _):
+        # If actual sends are being suppressed, we don't care about the event.
+        if not self.__is_suppressing_hardware:
+            if not self._backlight_workaround_task.is_killed:
+                self._backlight_workaround_task.kill()
+            self._backlight_workaround_task.restart()
+
+    # Use this while force-refreshing the LED state after a backlight update. Suppresses
+    # all messages for the backlight (because that's why we're here in the first place)
+    # and the standalone/hosted state (because it causes LED flicker and shouldn't be
+    # necessary to re-send).
+    @contextmanager
+    def __suppressing_backlight(self, is_suppressing_backlight=True):
+        old_suppressing_hardware = self.__is_suppressing_hardware
+        self.__is_suppressing_hardware = is_suppressing_backlight
+
+        try:
+            assert self.elements
+            backlight_sysex = self.elements.backlight_sysex
+            standalone_sysex = self.elements.standalone_sysex
+
+            with backlight_sysex.deferring_send(), standalone_sysex.deferring_send():
+                yield
+
+                # Hack to prevent any updates from actually getting sent.
+                backlight_sysex._deferred_message = None
+                standalone_sysex._deferred_message = None
+        finally:
+            self.__is_suppressing_hardware = old_suppressing_hardware

control_surface/colors.py

@@ -225,6 +225,9 @@ class Off:
         class On:
             On = TOGGLE_ON
 
+        class Unset:
+            On = TOGGLE_OFF
+
     class Mixer:
         ArmOn = GREEN_ON
         ArmOff = RED_ON

control_surface/configuration.py

@@ -61,8 +61,13 @@ class Configuration(NamedTuple):
     # may override this behavior.
     auto_arm: bool = False
 
-    # Whether to turn on the backlight. Can be toggled from utility mode.
-    backlight: bool = False
+    # Backlight on/off state (or `None` to leave it unmanaged) to be set at
+    # startup.
+    backlight: Optional[bool] = None
+
+    # Backlight on/off state (or `None` to leave it unmanaged) to be set at
+    # exit.
+    disconnect_backlight: Optional[bool] = None
 
     # Add a behavior when long pressing a clip (currently just "stop_track_clips" is available).
     clip_long_press_action: Optional[ClipSlotAction] = None
@@ -237,7 +242,8 @@ def get_configuration(song) -> Configuration:
 ElementOverride = Tuple[str, str, str]
 
 ##
-# Helpers for overriding elements. Keys are the physical key numbers on the SoftStep.
+# Helpers for overriding elements, see above for usage examples. Keys are the
+# physical key numbers on the SoftStep.
 
 
 # Override a key with an action.

control_surface/elements/sysex.py

@@ -1,10 +1,13 @@
-from typing import Collection, Optional, Tuple
+import logging
+from typing import Collection, Tuple
 
 from ableton.v2.control_surface.elements import ButtonElementMixin
 from ableton.v3.control_surface.elements import (
     SysexElement,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class SysexButtonElement(SysexElement, ButtonElementMixin):
     def is_momentary(self):
@@ -15,6 +18,9 @@ def is_momentary(self):
 # messages for a light, mode, etc. in the hardware. The element accepts True and False
 # as color values to trigger the on and off messages, respectively.
 class SysexToggleElement(SysexButtonElement):
+    # Fires after all the messages for the given on/off value have been sent.
+    __events__ = ("send_value",)
+
     def __init__(
         self,
         on_messages: Collection[Tuple[int, ...]],
@@ -26,12 +32,9 @@ def __init__(
         self._off_messages = off_messages
         assert len(on_messages) > 0 and len(off_messages) > 0
 
-        # Store the last value to `set_light` to avoid sending unnecessary messages.
-        self._last_value: Optional[bool] = None
-
         super().__init__(
             *a,
-            # Messages just get passed directly to send_value.
+            # Messages just get passed directly to the parent's send_value.
             send_message_generator=lambda msg: msg,
             optimized_send_midi=False,
             # Prevent mysterious crashes if there's mo
@@ -40,19 +43,18 @@ def __init__(
             **k,
         )
 
-    def _on_resource_received(self, client, *a, **k):
-        # Make sure we send our initial message. Since sending sysexes can cause
-        # momentary performance issues and other weirdness on the device, try to avoid
-        # disconnecting/reconnecting resources too often.
-        self._last_value = None
-        return super()._on_resource_received(client, *a, **k)
-
-    def set_light(self, value):
+    # This can get called via `set_light`, or from elsewhere within the framework.
+    def send_value(self, *a, **k):
+        assert len(a) == 1
+        value = a[0]
         assert isinstance(value, bool)
 
-        # Avoid re-sending these on every update.
-        if value != self._last_value:
-            messages = self._on_messages if value else self._off_messages
-            for message in messages:
-                self.send_value(message)
-        self._last_value = value
+        # Send multiple messages by calling the parent repeatedly.
+        messages = self._on_messages if value else self._off_messages
+        for message in messages:
+            super().send_value(message, *a[1:], **k)
+
+        self.notify_send_value(value)
+
+    def set_light(self, value):
+        self.send_value(value)