Integration test cleanup (#704)

* update more integration tests

* cleanup old skipped tests from migration

* missed file

* fix backend properties test

test/decorators.py

@@ -22,6 +22,20 @@
 from .unit.mock.fake_provider import FakeProvider
 
 
+def production_only(func):
+    """Decorator that runs a test only on production services."""
+
+    @wraps(func)
+    def _wrapper(self, *args, **kwargs):
+        if "dev" in self.dependencies.url:
+            raise SkipTest(
+                f"Skipping integration test. {self} is not supported on staging."
+            )
+        func(self, *args, **kwargs)
+
+    return _wrapper
+
+
 def run_fake_provider(func):
     """Decorator that runs a test using a fake provider."""
 

test/integration/test_backend.py

@@ -28,6 +28,7 @@
 from ..decorators import (
     IntegrationTestDependencies,
     integration_test_setup_with_backend,
+    production_only,
 )
 from ..ibm_test_case import IBMTestCase
 from ..utils import get_pulse_schedule, cancel_job
@@ -57,6 +58,7 @@ def test_backend_status(self):
         self.dependencies.provider.backends()
         self.assertTrue(self.backend.status().operational)
 
+    @production_only
     def test_backend_properties(self):
         """Check the properties of calibration of a real chip."""
         self.assertIsNotNone(self.backend.properties())
@@ -124,10 +126,9 @@ def test_sim_backend_options(self):
         self.assertTrue(backend_options["memory"])
         self.assertEqual(backend_options["foo"], "foo")
 
+    @production_only
     def test_paused_backend_warning(self):
         """Test that a warning is given when running jobs on a paused backend."""
-        if "dev" in self.dependencies.url:
-            raise SkipTest("Not supported in staging.")
         backend = self.dependencies.provider.get_backend("ibmq_qasm_simulator")
         paused_status = backend.status()
         paused_status.status_msg = "internal"
@@ -218,14 +219,15 @@ def test_too_many_qubits_in_circuit(self):
             instance=self.dependencies.instance, simulator=False
         )
         least_busy_backend = least_busy(backends)
-        self.assertTrue(least_busy_backend)
-
-        num = len(least_busy_backend.properties().qubits)
-        num_qubits = num + 1
-        circuit = QuantumCircuit(num_qubits, num_qubits)
-        with self.assertRaises(IBMBackendValueError) as err:
-            _ = least_busy_backend.run(circuit)
-        self.assertIn(
-            f"Circuit contains {num_qubits} qubits, but backend has only {num}.",
-            str(err.exception),
-        )
+        if least_busy_backend.properties():
+            self.assertTrue(least_busy_backend)
+
+            num = len(least_busy_backend.properties().qubits)
+            num_qubits = num + 1
+            circuit = QuantumCircuit(num_qubits, num_qubits)
+            with self.assertRaises(IBMBackendValueError) as err:
+                _ = least_busy_backend.run(circuit)
+            self.assertIn(
+                f"Circuit contains {num_qubits} qubits, but backend has only {num}.",
+                str(err.exception),
+            )

test/integration/test_ibm_job.py

@@ -32,6 +32,7 @@
 from ..decorators import (
     IntegrationTestDependencies,
     integration_test_setup_with_backend,
+    production_only,
 )
 from ..fake_account_client import BaseFakeAccountClient, CancelableFakeJob
 from ..ibm_test_case import IBMTestCase
@@ -169,10 +170,9 @@ def test_retrieve_pending_jobs(self):
         for job in pending_job_list:
             self.assertTrue(job.status() in [JobStatus.QUEUED, JobStatus.RUNNING])
 
+    @production_only
     def test_retrieve_running_error_jobs(self):
         """Test client side filtering with running and error jobs."""
-        if "dev" in self.dependencies.url:
-            raise SkipTest("Not supported in staging.")
         self.sim_job.wait_for_final_state()
         statuses = ["RUNNING", JobStatus.ERROR]
         job_list = self.provider.backend.jobs(

test/integration/test_ibm_job_attributes.py

@@ -26,14 +26,13 @@
 from qiskit_ibm_provider.api.clients.runtime import RuntimeClient
 from qiskit_ibm_provider.exceptions import (
     IBMBackendValueError,
-    IBMBackendApiProtocolError,
 )
 from qiskit_ibm_provider.job.exceptions import IBMJobFailureError
 from ..decorators import (
     IntegrationTestDependencies,
     integration_test_setup,
 )
-from ..fake_account_client import BaseFakeAccountClient, MissingFieldFakeJob
+from ..fake_account_client import BaseFakeAccountClient
 from ..ibm_test_case import IBMTestCase
 from ..utils import (
     most_busy_backend,
@@ -275,19 +274,6 @@ def test_esp_readout_default_value(self):
             delattr(self.sim_backend._configuration, "measure_esp_enabled")
             self.sim_backend._api_client = saved_api
 
-    @skip("not supported by api")
-    def test_esp_readout_enabled_not_used(self):
-        """Test that ESP readout is not used if user sets to ``False``, even if backend supports it."""
-        saved_api = self.sim_backend._api_client
-        try:
-            self.sim_backend._api_client = BaseFakeAccountClient()
-            setattr(self.sim_backend._configuration, "measure_esp_enabled", True)
-            job = self.sim_backend.run(self.bell, use_measure_esp=False)
-            self.assertEqual(job.backend_options()["use_measure_esp"], False)
-        finally:
-            delattr(self.sim_backend._configuration, "measure_esp_enabled")
-            self.sim_backend._api_client = saved_api
-
     def test_job_tags(self):
         """Test using job tags."""
         # Use a unique tag.
@@ -359,23 +345,3 @@ def test_cost_estimation(self):
         """Test cost estimation is returned correctly."""
         self.assertTrue(self.sim_job.usage_estimation)
         self.assertIn("quantum_seconds", self.sim_job.usage_estimation)
-
-    @skip("TODO refactor fake client")
-    def test_missing_required_fields(self):
-        """Test response data is missing required fields."""
-        saved_api = self.sim_backend._api_client
-        try:
-            self.sim_backend._api_client = BaseFakeAccountClient(
-                job_class=MissingFieldFakeJob
-            )
-            self.assertRaises(
-                IBMBackendApiProtocolError, self.sim_backend.run, self.bell
-            )
-        finally:
-            self.sim_backend._api_client = saved_api
-
-    @skip("not supported by api")
-    def test_client_version(self):
-        """Test job client version information."""
-        self.assertIsNotNone(self.sim_job.result().client_version)
-        self.assertIsNotNone(self.sim_job.client_version)

test/integration/test_ibm_provider.py

@@ -34,6 +34,7 @@
     IntegrationTestDependencies,
     integration_test_setup,
     integration_test_setup_with_backend,
+    production_only,
 )
 from ..ibm_test_case import IBMTestCase
 
@@ -140,10 +141,9 @@ def test_active_account_instance(self):
         )
         self.assertEqual(hgp.name, provider.active_account()["instance"])
 
+    @production_only
     def test_active_account_with_saved_instance(self):
         """Test active_account with a saved instance."""
-        if "dev" in self.dependencies.url:
-            self.skipTest("Test not supported on staging")
         hgp = self.provider._get_hgp()
         name = "foo"
         with temporary_account_config_file(

test/integration/test_serialization.py

@@ -13,17 +13,19 @@
 """Test serializing and deserializing data sent to the server."""
 
 from typing import Set, Any, Dict, Optional
-from unittest import SkipTest, skipIf, skip
+from unittest import SkipTest, skip
 
 import dateutil.parser
 from qiskit import transpile, schedule, QuantumCircuit
 from qiskit.circuit import Parameter
 from qiskit.test.reference_circuits import ReferenceCircuits
-from qiskit.version import VERSION as terra_version
-
 from qiskit_ibm_provider import least_busy
 from qiskit_ibm_provider.utils.json_encoder import IBMJsonEncoder
-from ..decorators import IntegrationTestDependencies, integration_test_setup
+from ..decorators import (
+    IntegrationTestDependencies,
+    integration_test_setup,
+    production_only,
+)
 from ..ibm_test_case import IBMTestCase
 from ..utils import cancel_job
 
@@ -112,6 +114,7 @@ def test_pulse_defaults(self):
             with self.subTest(backend=backend):
                 self._verify_data(backend.defaults().to_dict(), good_keys)
 
+    @production_only
     def test_backend_properties(self):
         """Test deserializing backend properties."""
         backends = self.dependencies.provider.backends(
@@ -185,7 +188,6 @@ def _verify_data(
                 }
         self.assertFalse(suspect_keys)
 
-    @skipIf(terra_version < "0.17", "Need Terra >= 0.17")
     def test_convert_complex(self):
         """Verify that real and complex ParameterExpressions are supported."""
         param = Parameter("test")

test/unit/test_ibm_job_states.py

@@ -21,7 +21,7 @@
 from concurrent import futures
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import suppress
-from unittest import mock, skip
+from unittest import mock
 from unittest.mock import MagicMock
 from typing import List, Any, Dict
 
@@ -39,9 +39,8 @@
 )
 from qiskit_ibm_provider.apiconstants import API_JOB_FINAL_STATES, ApiJobStatus
 
-# from qiskit_ibm_provider.exceptions import IBMBackendError
 from qiskit_ibm_provider.ibm_backend import IBMBackend
-from qiskit_ibm_provider.job.exceptions import IBMJobApiError, IBMJobInvalidStateError
+from qiskit_ibm_provider.job.exceptions import IBMJobInvalidStateError
 from ..jobtestcase import JobTestCase
 
 MOCKED_ERROR_RESULT: Dict[str, Any] = {
@@ -119,12 +118,6 @@ def setUp(self):
         self._current_api = None
         self._current_qjob = None
 
-    @skip("TODO refactor this case")
-    def test_unrecognized_status(self):
-        """Test unrecognized job state."""
-        with self.assertRaises(ValueError):
-            self.run_with_api(UnknownStatusAPI())
-
     def test_done_status(self):
         """Test job status progresses to done."""
         job = self.run_with_api(QueuedAPI())
@@ -218,54 +211,11 @@ def test_status_flow_for_cancellable_job(self):
         self._current_api.progress()
         self.assertEqual(job.status(), JobStatus.CANCELLED)
 
-    @skip("TODO refactor this case")
-    def test_status_flow_for_non_cancellable_job(self):
-        """Test job cannot be cancelled."""
-        job = self.run_with_api(NonCancellableAPI())
-
-        self.wait_for_initialization(job)
-        self.assertEqual(job.status(), JobStatus.RUNNING)
-
-        can_cancel = job.cancel()
-        self.assertFalse(can_cancel)
-
-        self._current_api.progress()
-        self.assertEqual(job.status(), JobStatus.RUNNING)
-
-    @skip("TODO refactor this case")
-    def test_status_flow_for_errored_cancellation(self):
-        """Test job cancel encounters an error."""
-        job = self.run_with_api(ErroredCancellationAPI())
-
-        self.wait_for_initialization(job)
-        self.assertEqual(job.status(), JobStatus.RUNNING)
-        can_cancel = job.cancel()
-        self.assertFalse(can_cancel)
-        self.assertEqual(job.status(), JobStatus.RUNNING)
-
     def test_status_flow_for_unable_to_run_valid_qobj(self):
         """Test API error while running a job."""
         with self.assertRaises(ApiError):
             self.run_with_api(UnavailableRunAPI())
 
-    @skip("TODO - fix test")
-    def test_api_throws_temporarily_but_job_is_finished(self):
-        """Test job finishes after encountering API error."""
-        job = self.run_with_api(ThrowingNonJobRelatedErrorAPI(errors_before_success=2))
-
-        # First time we query the server...
-        with self.assertRaises(ApiError):
-            # The error happens inside wait_for_initialization, the first time
-            # it calls to status() after INITIALIZING.
-            self.wait_for_initialization(job)
-
-        # Also an explicit second time...
-        with self.assertRaises(ApiError):
-            job.status()
-
-        # Now the API gets fixed and doesn't throw anymore.
-        self.assertEqual(job.status(), JobStatus.DONE)
-
     # TODO fix test case
     def test_error_while_running_job(self):
         """Test job failed."""
@@ -290,14 +240,6 @@ def test_cancelled_result(self):
             _ = job.result()
             self.assertEqual(job.status(), JobStatus.CANCELLED)
 
-    @skip("TODO - fix test")
-    def test_errored_result(self):
-        """Test getting results for a failed job."""
-        job = self.run_with_api(ThrowingGetJobAPI())
-        self.wait_for_initialization(job)
-        with self.assertRaises(IBMJobApiError):
-            job.result()
-
     def test_completed_result(self):
         """Test getting results for a completed job."""
         job = self.run_with_api(NonQueuedAPI())
@@ -330,18 +272,6 @@ def test_block_on_result_waiting_until_cancelled(self):
 
         self.assertEqual(job.status(), JobStatus.CANCELLED)
 
-    @skip("TODO - fix test")
-    def test_block_on_result_waiting_until_exception(self):
-        """Test getting API error while waiting for job results."""
-
-        job = self.run_with_api(ThrowingAPI())
-
-        with ThreadPoolExecutor() as executor:
-            executor.submit(_auto_progress_api, self._current_api)
-
-        with self.assertRaises(IBMJobApiError):
-            job.result()
-
     def test_never_complete_result_with_timeout(self):
         """Test timing out while waiting for job results."""
         job = self.run_with_api(NonQueuedAPI())
@@ -378,14 +308,6 @@ def test_only_final_states_cause_detailed_request(self):
                     else:
                         self.assertFalse(self._current_api.job_get.called)
 
-    @skip("TODO - fix test")
-    def test_no_kind_job(self):
-        """Test a job without the kind field."""
-        job = self.run_with_api(NoKindJobAPI())
-        with self.assertRaises(IBMJobInvalidStateError):
-            job.result()
-        self.assertFalse(job.circuits())
-
     def test_transpiling_status(self):
         """Test transpiling job state."""
         job = self.run_with_api(TranspilingStatusAPI())