From 11fa6c7b3f9cb033ff3a2b25356fdf38bc5794a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Fri, 20 Sep 2024 15:45:56 +0100 Subject: [PATCH] Introduce integration_test marker and update testing docs --- CONTRIBUTING.rst | 32 +++++++++++++---------------- setup.cfg | 2 ++ test/conftest.py | 12 +++++++++++ test/plugins/test_lyrics.py | 36 ++++++++------------------------- test/plugins/test_parentwork.py | 21 +++---------------- 5 files changed, 39 insertions(+), 64 deletions(-) create mode 100644 test/conftest.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0185049f70..b9593741c5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -378,28 +378,24 @@ Writing Tests Writing tests is done by adding or modifying files in folder `test`_. Take a look at `https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`_ -to get a basic view on how tests are written. We currently allow writing -tests with either `unittest`_ or `pytest`_. +to get a basic view on how tests are written. Since we are currently migrating +the tests from `unittest`_ to `pytest`_, new tests should be written using +`pytest`_. Contributions migrating existing tests are welcome! -Any tests that involve sending out network traffic e.g. an external API -call, should be skipped normally and run under our weekly `integration -test`_ suite. These tests can be useful in detecting external changes -that would affect ``beets``. In order to do this, simply add the -following snippet before the applicable test case: +External API requests under test should be mocked with `requests_mock`_, +However, we still want to know whether external APIs are up and that they +return expected responses, therefore we test them weekly with our `integration +test`_ suite. -.. code-block:: python +In order to add such a test, mark your test with the ``integration_test`` marker - @unittest.skipUnless( - os.environ.get('INTEGRATION_TEST', '0') == '1', - 'integration testing not enabled') +.. code-block:: python -If you do this, it is also advised to create a similar test that 'mocks' -the network call and can be run under normal circumstances by our CI and -others. See `unittest.mock`_ for more info. + @pytest.mark.integration_test + def test_external_api_call(): + ... -- **AVOID** using the ``start()`` and ``stop()`` methods of - ``mock.patch``, as they require manual cleanup. Use the annotation or - context manager forms instead. +This way, the test will be run only in the integration test suite. .. _Codecov: https://codecov.io/github/beetbox/beets .. _pytest-random: https://github.com/klrmn/pytest-random @@ -409,6 +405,6 @@ others. See `unittest.mock`_ for more info. .. _`https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`: https://github.com/beetbox/beets/blob/master/test/test_template.py#L224 .. _unittest: https://docs.python.org/3/library/unittest.html .. _integration test: https://github.com/beetbox/beets/actions?query=workflow%3A%22integration+tests%22 -.. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html +.. _requests-mock: https://requests-mock.readthedocs.io/en/latest/response.html .. _documentation: https://beets.readthedocs.io/en/stable/ .. _vim: https://www.vim.org/ diff --git a/setup.cfg b/setup.cfg index a45c49a81a..f694906307 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,8 @@ addopts = # show all skipped/failed/xfailed tests in the summary except passed -ra --strict-config +markers = + integration_test: mark a test as an integration test [coverage:run] data_file = .reports/coverage/data diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000000..8b29946ae2 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,12 @@ +import os + +import pytest + + +def pytest_runtest_setup(item: pytest.Item): + """Skip integration tests if INTEGRATION_TEST environment variable is not set.""" + if os.environ.get("INTEGRATION_TEST"): + return + + if next(item.iter_markers(name="integration_test"), None): + pytest.skip(f"INTEGRATION_TEST=1 required: {item.nodeid}") diff --git a/test/plugins/test_lyrics.py b/test/plugins/test_lyrics.py index 76c0112251..937e0a3cb1 100644 --- a/test/plugins/test_lyrics.py +++ b/test/plugins/test_lyrics.py @@ -21,6 +21,7 @@ from unittest.mock import MagicMock, patch import confuse +import pytest import requests from beets import logging @@ -335,10 +336,7 @@ def setUp(self): LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_backend_sources_ok(self): """Test default backends with songs known to exist in respective databases. @@ -351,10 +349,7 @@ def test_backend_sources_ok(self): res = backend.fetch(s["artist"], s["title"]) self.assertLyricsContentOk(s["title"], res) - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_google_sources_ok(self): """Test if lyrics present on websites registered in beets google custom search engine are correctly scraped. @@ -649,19 +644,13 @@ def setUp(self): self.plugin = lyrics.LyricsPlugin() tekstowo.config = self.plugin.config - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_normal(self): """Ensure we can fetch a song's lyrics in the ordinary case""" lyrics = tekstowo.fetch("Boy in Space", "u n eye") self.assertLyricsContentOk("u n eye", lyrics) - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_no_matching_results(self): """Ensure we fetch nothing if there are search results returned but no matches""" @@ -736,28 +725,19 @@ def setUp(self): self.plugin = lyrics.LyricsPlugin() lrclib.config = self.plugin.config - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_track_with_lyrics(self): lyrics = lrclib.fetch("Boy in Space", "u n eye", "Live EP", 160) self.assertLyricsContentOk("u n eye", lyrics) - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_instrumental_track(self): lyrics = lrclib.fetch( "Kelly Bailey", "Black Mesa Inbound", "Half Life 2 Soundtrack", 134 ) assert lyrics is None - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) + @pytest.mark.integration_test def test_nonexistent_track(self): lyrics = lrclib.fetch("blah", "blah", "blah", 999) assert lyrics is None diff --git a/test/plugins/test_parentwork.py b/test/plugins/test_parentwork.py index 9ce804091d..99267f6ffa 100644 --- a/test/plugins/test_parentwork.py +++ b/test/plugins/test_parentwork.py @@ -14,10 +14,10 @@ """Tests for the 'parentwork' plugin.""" -import os -import unittest from unittest.mock import patch +import pytest + from beets.library import Item from beets.test.helper import PluginTestCase from beetsplug import parentwork @@ -84,14 +84,11 @@ def mock_workid_response(mbid, includes): return p_work +@pytest.mark.integration_test class ParentWorkIntegrationTest(PluginTestCase): plugin = "parentwork" # test how it works with real musicbrainz data - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) def test_normal_case_real(self): item = Item( path="/file", @@ -106,10 +103,6 @@ def test_normal_case_real(self): item.load() assert item["mb_parentworkid"] == "32c8943f-1b27-3a23-8660-4567f4847c94" - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) def test_force_real(self): self.config["parentwork"]["force"] = True item = Item( @@ -127,10 +120,6 @@ def test_force_real(self): item.load() assert item["mb_parentworkid"] == "32c8943f-1b27-3a23-8660-4567f4847c94" - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) def test_no_force_real(self): self.config["parentwork"]["force"] = False item = Item( @@ -152,10 +141,6 @@ def test_no_force_real(self): # test different cases, still with Matthew Passion Ouverture or Mozart # requiem - @unittest.skipUnless( - os.environ.get("INTEGRATION_TEST", "0") == "1", - "integration testing not enabled", - ) def test_direct_parent_work_real(self): mb_workid = "2e4a3668-458d-3b2a-8be2-0b08e0d8243a" assert (