Introduce integration_test marker and update testing docs

CONTRIBUTING.rst

@@ -378,28 +378,24 @@ Writing Tests
 Writing tests is done by adding or modifying files in folder `test`_.
 Take a look at
 `https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`_
-to get a basic view on how tests are written. We currently allow writing
-tests with either `unittest`_ or `pytest`_.
+to get a basic view on how tests are written. Since we are currently migrating
+the tests from `unittest`_ to `pytest`_, new tests should be written using
+`pytest`_. Contributions migrating existing tests are welcome!
 
-Any tests that involve sending out network traffic e.g. an external API
-call, should be skipped normally and run under our weekly `integration
-test`_ suite. These tests can be useful in detecting external changes
-that would affect ``beets``. In order to do this, simply add the
-following snippet before the applicable test case:
+External API requests under test should be mocked with `requests_mock`_,
+However, we still want to know whether external APIs are up and that they
+return expected responses, therefore we test them weekly with our `integration
+test`_ suite.
 
-.. code-block:: python
+In order to add such a test, mark your test with the ``integration_test`` marker
 
-    @unittest.skipUnless(
-        os.environ.get('INTEGRATION_TEST', '0') == '1',
-        'integration testing not enabled')
+.. code-block:: python
 
-If you do this, it is also advised to create a similar test that 'mocks'
-the network call and can be run under normal circumstances by our CI and
-others. See `unittest.mock`_ for more info.
+  @pytest.mark.integration_test
+  def test_external_api_call():
+      ...
 
--  **AVOID** using the ``start()`` and ``stop()`` methods of
-   ``mock.patch``, as they require manual cleanup. Use the annotation or
-   context manager forms instead.
+This way, the test will be run only in the integration test suite.
 
 .. _Codecov: https://codecov.io/github/beetbox/beets
 .. _pytest-random: https://github.com/klrmn/pytest-random
@@ -409,6 +405,6 @@ others. See `unittest.mock`_ for more info.
 .. _`https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`: https://github.com/beetbox/beets/blob/master/test/test_template.py#L224
 .. _unittest: https://docs.python.org/3/library/unittest.html
 .. _integration test: https://github.com/beetbox/beets/actions?query=workflow%3A%22integration+tests%22
-.. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html
+.. _requests-mock: https://requests-mock.readthedocs.io/en/latest/response.html
 .. _documentation: https://beets.readthedocs.io/en/stable/
 .. _vim: https://www.vim.org/

setup.cfg

@@ -7,6 +7,8 @@ addopts =
     # show all skipped/failed/xfailed tests in the summary except passed
     -ra
     --strict-config
+markers =
+    integration_test: mark a test as an integration test
 
 [coverage:run]
 data_file = .reports/coverage/data

test/conftest.py

@@ -0,0 +1,12 @@
+import os
+
+import pytest
+
+
+def pytest_runtest_setup(item: pytest.Item):
+    """Skip integration tests if INTEGRATION_TEST environment variable is not set."""
+    if os.environ.get("INTEGRATION_TEST"):
+        return
+
+    if next(item.iter_markers(name="integration_test"), None):
+        pytest.skip(f"INTEGRATION_TEST=1 required: {item.nodeid}")

test/plugins/test_lyrics.py

@@ -21,6 +21,7 @@
 from unittest.mock import MagicMock, patch
 
 import confuse
+import pytest
 import requests
 
 from beets import logging
@@ -335,10 +336,7 @@ def setUp(self):
         LyricsGoogleBaseTest.setUp(self)
         self.plugin = lyrics.LyricsPlugin()
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_backend_sources_ok(self):
         """Test default backends with songs known to exist in respective
         databases.
@@ -351,10 +349,7 @@ def test_backend_sources_ok(self):
                 res = backend.fetch(s["artist"], s["title"])
                 self.assertLyricsContentOk(s["title"], res)
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_google_sources_ok(self):
         """Test if lyrics present on websites registered in beets google custom
         search engine are correctly scraped.
@@ -649,19 +644,13 @@ def setUp(self):
         self.plugin = lyrics.LyricsPlugin()
         tekstowo.config = self.plugin.config
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_normal(self):
         """Ensure we can fetch a song's lyrics in the ordinary case"""
         lyrics = tekstowo.fetch("Boy in Space", "u n eye")
         self.assertLyricsContentOk("u n eye", lyrics)
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_no_matching_results(self):
         """Ensure we fetch nothing if there are search results
         returned but no matches"""
@@ -736,28 +725,19 @@ def setUp(self):
         self.plugin = lyrics.LyricsPlugin()
         lrclib.config = self.plugin.config
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_track_with_lyrics(self):
         lyrics = lrclib.fetch("Boy in Space", "u n eye", "Live EP", 160)
         self.assertLyricsContentOk("u n eye", lyrics)
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_instrumental_track(self):
         lyrics = lrclib.fetch(
             "Kelly Bailey", "Black Mesa Inbound", "Half Life 2 Soundtrack", 134
         )
         assert lyrics is None
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
+    @pytest.mark.integration_test
     def test_nonexistent_track(self):
         lyrics = lrclib.fetch("blah", "blah", "blah", 999)
         assert lyrics is None

test/plugins/test_parentwork.py

@@ -14,10 +14,10 @@
 
 """Tests for the 'parentwork' plugin."""
 
-import os
-import unittest
 from unittest.mock import patch
 
+import pytest
+
 from beets.library import Item
 from beets.test.helper import PluginTestCase
 from beetsplug import parentwork
@@ -84,14 +84,11 @@ def mock_workid_response(mbid, includes):
         return p_work
 
 
+@pytest.mark.integration_test
 class ParentWorkIntegrationTest(PluginTestCase):
     plugin = "parentwork"
 
     # test how it works with real musicbrainz data
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
     def test_normal_case_real(self):
         item = Item(
             path="/file",
@@ -106,10 +103,6 @@ def test_normal_case_real(self):
         item.load()
         assert item["mb_parentworkid"] == "32c8943f-1b27-3a23-8660-4567f4847c94"
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
     def test_force_real(self):
         self.config["parentwork"]["force"] = True
         item = Item(
@@ -127,10 +120,6 @@ def test_force_real(self):
         item.load()
         assert item["mb_parentworkid"] == "32c8943f-1b27-3a23-8660-4567f4847c94"
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
     def test_no_force_real(self):
         self.config["parentwork"]["force"] = False
         item = Item(
@@ -152,10 +141,6 @@ def test_no_force_real(self):
     # test different cases, still with Matthew Passion Ouverture or Mozart
     # requiem
 
-    @unittest.skipUnless(
-        os.environ.get("INTEGRATION_TEST", "0") == "1",
-        "integration testing not enabled",
-    )
     def test_direct_parent_work_real(self):
         mb_workid = "2e4a3668-458d-3b2a-8be2-0b08e0d8243a"
         assert (