AEA-710 reenable tests and update marks

.github/workflows/workflow.yml

@@ -54,7 +54,7 @@ jobs:
         sudo apt-get install -y protobuf-compiler
     - name: Integrational tests and coverage
       run: |
-        tox -e py3.8 -- --aea-loop sync -m 'integration and not unstable and not ethereum'
+        tox -e py3.8 -- --aea-loop sync -m 'integration and not unstable and not ledger'
 
   common_checks:
     runs-on: ubuntu-latest
@@ -128,9 +128,9 @@ jobs:
           pip install pipenv
           pip install tox
       - name: Integration tests
-        run: tox -e py3.7 -- -m 'integration and not unstable and not ethereum'
+        run: tox -e py3.7 -- -m 'integration and not unstable and not ledger'
 
-  integration_checks_eth:
+  integration_checks_ledger:
     continue-on-error: True
     runs-on: ubuntu-latest
 
@@ -149,7 +149,7 @@ jobs:
           pip install pipenv
           pip install tox
       - name: Integration tests
-        run: tox -e py3.7 -- -m 'integration and not unstable and ethereum'
+        run: tox -e py3.7 -- -m 'integration and not unstable and ledger'
         continue-on-error: true
       - name: Force green exit
         run: exit 0

aea/test_tools/test_cases.py

@@ -18,6 +18,7 @@
 # ------------------------------------------------------------------------------
 
 """This module contains test case classes based on pytest for AEA end-to-end testing."""
+import copy
 import os
 import random
 import shutil
@@ -253,7 +254,7 @@ def difference_to_fetched_agent(cls, public_id: str, agent_name: str) -> List[st
 
         def is_allowed_diff_in_agent_config(
             path_to_fetched_aea, path_to_manually_created_aea
-        ) -> bool:
+        ) -> Tuple[bool, Dict[str, str], Dict[str, str]]:
             with open(
                 os.path.join(path_to_fetched_aea, "aea-config.yaml"), "r"
             ) as file:
@@ -262,12 +263,20 @@ def is_allowed_diff_in_agent_config(
                 os.path.join(path_to_manually_created_aea, "aea-config.yaml"), "r"
             ) as file:
                 content2 = yaml.full_load(file)
-            diff_count = 0
-            for key, value in content1.items():
-                if content2[key] != value:
-                    diff_count += 1
-            # allow diff in aea_version, author, description and version
-            return diff_count <= 4
+            content1c = copy.deepcopy(content1)
+            for key, value in content1c.items():
+                if content2[key] == value:
+                    content1.pop(key)
+                    content2.pop(key)
+            allowed_diff_keys = ["aea_version", "author", "description", "version"]
+            result = all([key in allowed_diff_keys for key in content1.keys()])
+            result = result and all(
+                [key in allowed_diff_keys for key in content2.keys()]
+            )
+            if result:
+                return result, {}, {}
+            else:
+                return result, content1, content2
 
         path_to_manually_created_aea = os.path.join(cls.t, agent_name)
         new_cwd = os.path.join(cls.t, "fetch_dir")
@@ -279,10 +288,15 @@ def is_allowed_diff_in_agent_config(
             cls.run_cli_command("fetch", "--local", public_id, "--alias", agent_name)
         comp = dircmp(path_to_manually_created_aea, path_to_fetched_aea)
         file_diff = comp.diff_files
-        if is_allowed_diff_in_agent_config(
+        result, diff1, diff2 = is_allowed_diff_in_agent_config(
             path_to_fetched_aea, path_to_manually_created_aea
-        ):
+        )
+        if result:
             file_diff.remove("aea-config.yaml")  # won't match!
+        else:
+            file_diff.append(
+                "Difference in aea-config.yaml: " + str(diff1) + " vs. " + str(diff2)
+            )
         try:
             shutil.rmtree(new_cwd)
         except (OSError, IOError):

docs/erc1155-skills.md

@@ -81,7 +81,7 @@ aea add-key ethereum eth_private_key.txt
 In another terminal, fetch the AEA that will get some tokens from the deployer.
 
 ``` bash
-aea fetch fetchai/erc1155_client:0.7.0
+aea fetch fetchai/erc1155_client:0.8.0
 cd erc1155_client
 aea install
 ```
@@ -96,7 +96,7 @@ aea create erc1155_client
 cd erc1155_client
 aea add connection fetchai/oef:0.5.0
 aea add connection fetchai/ledger:0.1.0
-aea add skill fetchai/erc1155_client:0.6.0
+aea add skill fetchai/erc1155_client:0.7.0
 aea install
 aea config set agent.default_connection fetchai/oef:0.5.0
 ```

packages/fetchai/agents/erc1155_client/aea-config.yaml

@@ -1,6 +1,6 @@
 agent_name: erc1155_client
 author: fetchai
-version: 0.7.0
+version: 0.8.0
 description: An AEA to interact with the ERC1155 deployer AEA
 license: Apache-2.0
 aea_version: '>=0.5.0, <0.6.0'
@@ -20,7 +20,7 @@ protocols:
 - fetchai/oef_search:0.3.0
 - fetchai/signing:0.1.0
 skills:
-- fetchai/erc1155_client:0.6.0
+- fetchai/erc1155_client:0.7.0
 - fetchai/error:0.3.0
 default_connection: fetchai/oef:0.5.0
 default_ledger: ethereum

packages/fetchai/skills/erc1155_client/skill.yaml

@@ -1,7 +1,8 @@
 name: erc1155_client
 author: fetchai
-version: 0.6.0
-description: The weather client skill implements the skill to purchase weather data.
+version: 0.7.0
+description: The erc1155 client interacts with the erc1155 deployer to conduct an
+  atomic swap.
 license: Apache-2.0
 aea_version: '>=0.5.0, <0.6.0'
 fingerprint:
@@ -14,9 +15,12 @@ fingerprint_ignore_patterns: []
 contracts:
 - fetchai/erc1155:0.6.0
 protocols:
+- fetchai/contract_api:0.1.0
 - fetchai/default:0.3.0
 - fetchai/fipa:0.4.0
+- fetchai/ledger_api:0.1.0
 - fetchai/oef_search:0.3.0
+- fetchai/signing:0.1.0
 skills: []
 behaviours:
   search:

packages/hashes.csv

@@ -2,7 +2,7 @@ fetchai/agents/aries_alice,Qma9e8EXGU3bKQMRxwJdNvUDdPCHEpDUUNYqGyT2KtrNpj
 fetchai/agents/aries_faber,QmPSsKEqfh26murXXdZ8kocuAJtNwNne6jjVVkDAQnSthb
 fetchai/agents/car_data_buyer,QmcBd8VLx11U2nDPBt4W4YyRToUJpEQytfFqHDTJDZA5AR
 fetchai/agents/car_detector,QmXMjcvVo8Qg7a8u3GadWWtPHnrzFKkxkTHap6JRjDHXm4
-fetchai/agents/erc1155_client,QmcNrPxgokFn2WUWed5vsnRxCgbLPKXsERbTs6hVE7NLcm
+fetchai/agents/erc1155_client,QmYSpr6ZTEXWheNxUD5Z7dXUS1eZxbe498JuHqwwjKs888
 fetchai/agents/erc1155_deployer,QmevosZhB78HTPQAb62v8hLCdtcSqdoSQnKWwKkbXT55L4
 fetchai/agents/generic_buyer,QmPAdWvKuw3VFxxQi9NkMPAC4ymAwVSftaYbc5upBTtPtf
 fetchai/agents/generic_seller,QmUF18HoArCHf6mLdXjq1zXCuJKY7JwXXSYTdfsWCwPWKn
@@ -52,7 +52,7 @@ fetchai/skills/aries_faber,QmcqRhcdZ3v42bd9gX2wMVB81Xq7tztumknxcWeKYJm6cB
 fetchai/skills/carpark_client,Qme1o7xwV9mRv9yBzTRxbEqxrz5J14nyu5MKYaMqJMb5nq
 fetchai/skills/carpark_detection,QmQByZH6G6b4PmU2REiny33GcRcpo9aYnZAhYcUiBff9ME
 fetchai/skills/echo,QmeSr4j8W9enijZvgeE3vXeWcEj9sS8fo6vNFRpyAMnZey
-fetchai/skills/erc1155_client,Qmb2EiLT8ycav2fKsuZb2DhGiNzgoZym1LX8dYCSe7V9qQ
+fetchai/skills/erc1155_client,QmRfnZPPh5a8Z7nPf4AcjA9VGei1yYLEqDCDnxgamu4mHz
 fetchai/skills/erc1155_deploy,QmRS56TANANu3yw8mk4cBf7iiA4SQ3d1ZPnSLFdo4XrcTE
 fetchai/skills/error,QmVirmcRGj6bc2i6iJZ2zoWGCfsCZMoGmZAXYq5aaYAqNb
 fetchai/skills/generic_buyer,QmabHUAbLja1hsHU8p7M6TSx9nNsNAE9rAj31nwV1LX1Sm

pytest.ini

@@ -5,8 +5,7 @@ log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno
 log_cli_date_format=%Y-%m-%d %H:%M:%S
 
 markers =
-    integration: marks end-to-end tests which require the oef
-    unstable: marks test as unstable
-    network: marks tests which require internet access
-    ethereum: marks tests which require ethereum test network
+    integration: marks end-to-end tests which require the oef, ledger or other network services
+    unstable: marks test as unstable (won't be run in CI)
+    ledger: marks tests which require ledger test networks (ethereum, cosmos, fetchai); these tests should also be marked 'integration'
     flaky: marks tests which are flaky and worth re-running

tests/test_cli/test_add/test_connection.py

@@ -475,10 +475,10 @@ def teardown_class(cls):
             pass
 
 
-@pytest.mark.integration
 class TestAddConnectionFromRemoteRegistry(AEATestCaseEmpty):
     """Test case for add connection from Registry command."""
 
+    @pytest.mark.integration
     @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
     def test_add_connection_from_remote_registry_positive(self):
         """Test add connection from Registry positive result."""

tests/test_cli/test_add/test_contract.py

@@ -56,10 +56,10 @@ def test_add_contract_positive(self, *mocks):
         self.assertEqual(result.exit_code, 0)
 
 
-@pytest.mark.integration
 class TestAddContractFromRemoteRegistry(AEATestCaseEmpty):
     """Test case for add contract from Registry command."""
 
+    @pytest.mark.integration
     @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
     def test_add_contract_from_remote_registry_positive(self):
         """Test add contract from Registry positive result."""

tests/test_cli/test_add/test_protocol.py

@@ -471,10 +471,10 @@ def teardown_class(cls):
             pass
 
 
-@pytest.mark.integration
 class TestAddProtocolFromRemoteRegistry(AEATestCaseEmpty):
     """Test case for add protocol from Registry command."""
 
+    @pytest.mark.integration
     @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
     def test_add_protocol_from_remote_registry_positive(self):
         """Test add protocol from Registry positive result."""

tests/test_cli/test_add/test_skill.py

@@ -488,18 +488,18 @@ class TestAddSkillWithContractsDeps(AEATestCaseEmpty):
 
     def test_add_skill_with_contracts_positive(self):
         """Test add skill with contract dependencies positive result."""
-        self.add_item("skill", "fetchai/erc1155_client:0.6.0")
+        self.add_item("skill", "fetchai/erc1155_client:0.7.0")
 
         contracts_path = os.path.join(self.agent_name, "vendor", "fetchai", "contracts")
         contracts_folders = os.listdir(contracts_path)
         contract_dependency_name = "erc1155"
         assert contract_dependency_name in contracts_folders
 
 
-@pytest.mark.integration
 class TestAddSkillFromRemoteRegistry(AEATestCaseEmpty):
     """Test case for add skill from Registry command."""
 
+    @pytest.mark.integration
     @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
     def test_add_skill_from_remote_registry_positive(self):
         """Test add skill from Registry positive result."""

tests/test_cli/test_fetch.py

@@ -143,10 +143,10 @@ def test__is_version_correct_negative(self):
         self.assertFalse(result)
 
 
-@pytest.mark.integration
 class TestFetchFromRemoteRegistry(AEATestCaseMany):
     """Test case for fetch agent command from Registry."""
 
+    @pytest.mark.integration
     @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
     def test_fetch_agent_from_remote_registry_positive(self):
         """Test fetch agent from Registry for positive result."""

tests/test_cli/test_launch.py

@@ -40,7 +40,7 @@
 
 from tests.common.pexpect_popen import PexpectWrapper
 
-from ..conftest import AUTHOR, CLI_LOG_OPTION, CUR_PATH
+from ..conftest import AUTHOR, CLI_LOG_OPTION, CUR_PATH, MAX_FLAKY_RERUNS
 
 logger = logging.getLogger(__name__)
 
@@ -135,7 +135,7 @@ def test_exit_code_equal_to_zero(self):
             )
 
 
-@pytest.mark.unstable
+@pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
 class TestLaunchWithOneFailingAgent(BaseLaunchTestCase):
     """Test aea launch when there is a failing agent.."""
 

tests/test_cli_gui/test_run_agent.py

@@ -34,10 +34,10 @@
 from aea.test_tools.constants import DEFAULT_AUTHOR
 
 from .test_base import TempCWD, create_app
-from ..conftest import CUR_PATH, skip_test_windows
+from ..conftest import CUR_PATH, MAX_FLAKY_RERUNS, skip_test_windows
 
 
-@pytest.mark.unstable
+@pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
 @skip_test_windows
 def test_create_and_run_agent():
     """Test for running and agent, reading TTY and errors."""

tests/test_crypto/test_cosmos.py

@@ -90,7 +90,8 @@ def test_generate_nonce():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_construct_sign_and_submit_transfer_transaction():
     """Test the construction, signing and submitting of a transfer transaction."""
     account = CosmosCrypto(COSMOS_PRIVATE_KEY_PATH)
@@ -141,7 +142,8 @@ def test_construct_sign_and_submit_transfer_transaction():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_get_balance():
     """Test the balance is zero for a new account."""
     cosmos_api = CosmosApi(**COSMOS_TESTNET_CONFIG)
@@ -154,7 +156,8 @@ def test_get_balance():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_get_wealth_positive(caplog):
     """Test the balance is zero for a new account."""
     with caplog.at_level(logging.DEBUG, logger="aea.crypto.cosmos"):

tests/test_crypto/test_ethereum.py

@@ -113,7 +113,8 @@ def test_api_none():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_get_balance():
     """Test the balance is zero for a new account."""
     ethereum_api = EthereumApi(**ETHEREUM_TESTNET_CONFIG)
@@ -125,9 +126,9 @@ def test_get_balance():
     assert balance > 0, "Existing account has no balance."
 
 
-@pytest.mark.unstable  # TODO: fix
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
 @pytest.mark.integration
+@pytest.mark.ledger
 def test_construct_sign_and_submit_transfer_transaction():
     """Test the construction, signing and submitting of a transfer transaction."""
     account = EthereumCrypto(private_key_path=ETHEREUM_PRIVATE_KEY_PATH)
@@ -179,7 +180,8 @@ def test_construct_sign_and_submit_transfer_transaction():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_get_wealth_positive(caplog):
     """Test the balance is zero for a new account."""
     with caplog.at_level(logging.DEBUG, logger="aea.crypto.ethereum"):

tests/test_crypto/test_fetchai.py

@@ -97,7 +97,8 @@ def test_generate_nonce():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_construct_sign_and_submit_transfer_transaction():
     """Test the construction, signing and submitting of a transfer transaction."""
     account = FetchAICrypto(FETCHAI_PRIVATE_KEY_PATH)
@@ -149,7 +150,8 @@ def test_construct_sign_and_submit_transfer_transaction():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_get_balance():
     """Test the balance is zero for a new account."""
     fetchai_api = FetchAIApi(**FETCHAI_TESTNET_CONFIG)
@@ -162,7 +164,8 @@ def test_get_balance():
 
 
 @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS)
-@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.ledger
 def test_get_wealth_positive(caplog):
     """Test the balance is zero for a new account."""
     with caplog.at_level(logging.DEBUG, logger="aea.crypto.fetchai"):

tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md

@@ -23,7 +23,7 @@ aea generate-key ethereum
 aea add-key ethereum eth_private_key.txt
 ```
 ``` bash
-aea fetch fetchai/erc1155_client:0.7.0
+aea fetch fetchai/erc1155_client:0.8.0
 cd erc1155_client
 aea install
 ```
@@ -32,7 +32,7 @@ aea create erc1155_client
 cd erc1155_client
 aea add connection fetchai/oef:0.5.0
 aea add connection fetchai/ledger:0.1.0
-aea add skill fetchai/erc1155_client:0.6.0
+aea add skill fetchai/erc1155_client:0.7.0
 aea install
 aea config set agent.default_connection fetchai/oef:0.5.0
 ```