From 5fcd18c4b52513729d6045c32ef39c068151c12c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 20 Nov 2019 10:05:23 -0800 Subject: [PATCH 01/51] [TESTS] Add tests of requestData This will allow us to push the GetData creation inside P2PDataStorage safely. --- .../p2p/peers/getdata/RequestDataHandler.java | 4 +- .../RequestDataHandlerRequestDataTest.java | 249 ++++++++++++++++++ .../bisq/network/p2p/storage/TestState.java | 5 +- .../mocks/PersistableNetworkPayloadStub.java | 12 +- 4 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index a116d171b7d..161e44e7e99 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -38,6 +38,7 @@ import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.network.NetworkPayload; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; @@ -92,7 +93,8 @@ public interface Listener { private final PeerManager peerManager; private final Listener listener; private Timer timeoutTimer; - private final int nonce = new Random().nextInt(); + @VisibleForTesting + final int nonce = new Random().nextInt(); private boolean stopped; diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java new file mode 100644 index 00000000000..e757ce07f04 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java @@ -0,0 +1,249 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.peers.getdata; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.PeerManager; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.TestState; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; + +import com.google.common.util.concurrent.SettableFuture; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Set; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + + + +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.*; + +public class RequestDataHandlerRequestDataTest { + private TestState testState; + + @Mock + NetworkNode networkNode; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); + + // Set up basic capabilities to ensure message contains it + Capabilities.app.addAll(Capability.MEDIATION); + } + + /** + * Returns true if the target bytes are found in the container set. + */ + private boolean byteSetContains(Set container, byte[] target) { + // Set.contains() doesn't do a deep compare, so generate a Set so equals() does what + // we want + Set translatedContainer = + P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(container); + + return translatedContainer.contains(new P2PDataStorage.ByteArray(target)); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest + @Test + public void requestData_EmptyP2PDataStore_PreliminaryGetDataRequest() { + SettableFuture sendFuture = mock(SettableFuture.class); + + final ArgumentCaptor captor = ArgumentCaptor.forClass(PreliminaryGetDataRequest.class); + when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); + + RequestDataHandler handler = new RequestDataHandler( + this.networkNode, + this.testState.getMockedStorage(), + mock(PeerManager.class), + mock(RequestDataHandler.Listener.class) + ); + + handler.requestData(this.localNodeAddress, true); + + // expect empty message since p2pDataStore is empty + PreliminaryGetDataRequest getDataRequest = captor.getValue(); + + Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); + Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); + } + + // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest + @Test + public void requestData_EmptyP2PDataStore_GetUpdatedDataRequest() { + SettableFuture sendFuture = mock(SettableFuture.class); + + final ArgumentCaptor captor = ArgumentCaptor.forClass(GetUpdatedDataRequest.class); + when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); + + RequestDataHandler handler = new RequestDataHandler( + this.networkNode, + this.testState.getMockedStorage(), + mock(PeerManager.class), + mock(RequestDataHandler.Listener.class) + ); + + handler.requestData(this.localNodeAddress, false); + + // expect empty message since p2pDataStore is empty + GetUpdatedDataRequest getDataRequest = captor.getValue(); + + Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); + Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); + Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); + } + + // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates + // correct GetDataRequestMessage with both sets of keys. + @Test + public void requestData_FilledP2PDataStore_PreliminaryGetDataRequest() throws NoSuchAlgorithmException { + SettableFuture sendFuture = mock(SettableFuture.class); + + PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); + ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); + + this.testState.getMockedStorage().addPersistableNetworkPayload( + toAdd1, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addPersistableNetworkPayload( + toAdd2, this.localNodeAddress, false, false, false, false); + + this.testState.getMockedStorage().addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); + this.testState.getMockedStorage().addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + + final ArgumentCaptor captor = ArgumentCaptor.forClass(PreliminaryGetDataRequest.class); + when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); + + RequestDataHandler handler = new RequestDataHandler( + this.networkNode, + this.testState.getMockedStorage(), + mock(PeerManager.class), + mock(RequestDataHandler.Listener.class) + ); + + handler.requestData(this.localNodeAddress, true); + + // expect empty message since p2pDataStore is empty + PreliminaryGetDataRequest getDataRequest = captor.getValue(); + + Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); + Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); + } + + // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates + // correct GetDataRequestMessage with both sets of keys. + @Test + public void requestData_FilledP2PDataStore_GetUpdatedDataRequest() throws NoSuchAlgorithmException { + SettableFuture sendFuture = mock(SettableFuture.class); + + PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); + ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); + + this.testState.getMockedStorage().addPersistableNetworkPayload( + toAdd1, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addPersistableNetworkPayload( + toAdd2, this.localNodeAddress, false, false, false, false); + + this.testState.getMockedStorage().addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); + this.testState.getMockedStorage().addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + + final ArgumentCaptor captor = ArgumentCaptor.forClass(GetUpdatedDataRequest.class); + when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); + + RequestDataHandler handler = new RequestDataHandler( + this.networkNode, + this.testState.getMockedStorage(), + mock(PeerManager.class), + mock(RequestDataHandler.Listener.class) + ); + + handler.requestData(this.localNodeAddress, false); + + // expect empty message since p2pDataStore is empty + GetUpdatedDataRequest getDataRequest = captor.getValue(); + + Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); + Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); + Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); + } +} diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index c4ceab5ca2a..88d465c5734 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -51,6 +51,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import lombok.Getter; + import org.junit.Assert; import org.mockito.ArgumentCaptor; @@ -65,6 +67,7 @@ public class TestState { static final int MAX_SEQUENCE_NUMBER_MAP_SIZE_BEFORE_PURGE = 5; + @Getter P2PDataStorage mockedStorage; final Broadcaster mockBroadcaster; @@ -74,7 +77,7 @@ public class TestState { private final ProtectedDataStoreService protectedDataStoreService; final ClockFake clockFake; - TestState() { + public TestState() { this.mockBroadcaster = mock(Broadcaster.class); this.mockSeqNrStorage = mock(Storage.class); this.clockFake = new ClockFake(); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java b/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java index 6ab73155480..057cb2c42b8 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/mocks/PersistableNetworkPayloadStub.java @@ -28,9 +28,19 @@ */ public class PersistableNetworkPayloadStub implements PersistableNetworkPayload { private final boolean hashSizeValid; + private final byte[] hash; public PersistableNetworkPayloadStub(boolean hashSizeValid) { + this(hashSizeValid, new byte[] { 1 }); + } + + public PersistableNetworkPayloadStub(byte[] hash) { + this(true, hash); + } + + private PersistableNetworkPayloadStub(boolean hashSizeValid, byte[] hash) { this.hashSizeValid = hashSizeValid; + this.hash = hash; } @Override @@ -40,7 +50,7 @@ public protobuf.PersistableNetworkPayload toProtoMessage() { @Override public byte[] getHash() { - return new byte[] { 1 }; + return hash; } @Override From 1e814d9f1d1cc16a59665a7390a603f1fec2040d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 20 Nov 2019 10:38:04 -0800 Subject: [PATCH 02/51] [REFACTOR] Introduce buildGetDataRequest variants As part of changing the GetData path, we want to move all creation and processing of GetData messages inside P2PDataStorage. This will allow easier unit testing of the behavior as well as cleaner code in the request handlers that can just focus on nonces, connections, etc. --- .../p2p/peers/getdata/RequestDataHandler.java | 20 +-------- .../network/p2p/storage/P2PDataStorage.java | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index 161e44e7e99..10b1b7ae905 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -49,7 +49,6 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -126,25 +125,10 @@ void requestData(NodeAddress nodeAddress, boolean isPreliminaryDataRequest) { if (!stopped) { GetDataRequest getDataRequest; - // We collect the keys of the PersistableNetworkPayload items so we exclude them in our request. - // PersistedStoragePayload items don't get removed, so we don't have an issue with the case that - // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would - // miss that event if we do not load the full set or use some delta handling. - Set excludedKeys = dataStorage.getAppendOnlyDataStoreMap().keySet().stream() - .map(e -> e.bytes) - .collect(Collectors.toSet()); - - Set excludedKeysFromPersistedEntryMap = dataStorage.getMap().keySet() - .stream() - .map(e -> e.bytes) - .collect(Collectors.toSet()); - - excludedKeys.addAll(excludedKeysFromPersistedEntryMap); - if (isPreliminaryDataRequest) - getDataRequest = new PreliminaryGetDataRequest(nonce, excludedKeys); + getDataRequest = dataStorage.buildPreliminaryGetDataRequest(nonce); else - getDataRequest = new GetUpdatedDataRequest(networkNode.getNodeAddress(), nonce, excludedKeys); + getDataRequest = dataStorage.buildGetUpdatedDataRequest(networkNode.getNodeAddress(), nonce); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index c0a22c691b2..a66557ccd3c 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -25,6 +25,8 @@ import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.BroadcastHandler; import bisq.network.p2p.peers.Broadcaster; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.messages.AddDataMessage; import bisq.network.p2p.storage.messages.AddOncePayload; import bisq.network.p2p.storage.messages.AddPersistableNetworkPayloadMessage; @@ -177,6 +179,46 @@ public synchronized void readFromResources(String postFix) { map.putAll(protectedDataStoreService.getMap()); } + /////////////////////////////////////////////////////////////////////////////////////////// + // RequestData API + /////////////////////////////////////////////////////////////////////////////////////////// + + /** + * Returns a PreliminaryGetDataRequest that can be sent to a peer node to request missing Payload data. + */ + public PreliminaryGetDataRequest buildPreliminaryGetDataRequest(int nonce) { + return new PreliminaryGetDataRequest(nonce, this.getKnownPayloadHashes()); + } + + /** + * Returns a GetUpdatedDataRequest that can be sent to a peer node to request missing Payload data. + */ + public GetUpdatedDataRequest buildGetUpdatedDataRequest(NodeAddress senderNodeAddress, int nonce) { + return new GetUpdatedDataRequest(senderNodeAddress, nonce, this.getKnownPayloadHashes()); + } + + /** + * Returns the set of known payload hashes. This is used in the GetData path to request missing data from peer nodes + */ + private Set getKnownPayloadHashes() { + // We collect the keys of the PersistableNetworkPayload items so we exclude them in our request. + // PersistedStoragePayload items don't get removed, so we don't have an issue with the case that + // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would + // miss that event if we do not load the full set or use some delta handling. + Set excludedKeys =this.appendOnlyDataStoreService.getMap().keySet().stream() + .map(e -> e.bytes) + .collect(Collectors.toSet()); + + Set excludedKeysFromPersistedEntryMap = this.map.keySet() + .stream() + .map(e -> e.bytes) + .collect(Collectors.toSet()); + + excludedKeys.addAll(excludedKeysFromPersistedEntryMap); + + return excludedKeys; + } + /////////////////////////////////////////////////////////////////////////////////////////// // API From a927ed42ac38035dc9d1b024dacec737a3bf443a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 20 Nov 2019 10:54:53 -0800 Subject: [PATCH 03/51] [TESTS] Add tests of new RequestData APIs These are identical test cases to the requestHandler tests, but with much fewer dependencies. The requestHandler tests will eventually be deleted, but they are going to remain throughout development as an extra safety net. --- .../P2PDataStorageRequestDataTest.java | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java new file mode 100644 index 00000000000..55d16344d64 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java @@ -0,0 +1,175 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Set; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.*; + +public class P2PDataStorageRequestDataTest { + private TestState testState; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + + // Set up basic capabilities to ensure message contains it + Capabilities.app.addAll(Capability.MEDIATION); + } + + /** + * Returns true if the target bytes are found in the container set. + */ + private boolean byteSetContains(Set container, byte[] target) { + // Set.contains() doesn't do a deep compare, so generate a Set so equals() does what + // we want + Set translatedContainer = + P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(container); + + return translatedContainer.contains(new P2PDataStorage.ByteArray(target)); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest + @Test + public void buildPreliminaryGetDataRequest_EmptyP2PDataStore() { + PreliminaryGetDataRequest getDataRequest = this.testState.mockedStorage.buildPreliminaryGetDataRequest(1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); + } + + // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest + @Test + public void buildGetUpdatedDataRequest_EmptyP2PDataStore() { + GetUpdatedDataRequest getDataRequest = + this.testState.mockedStorage.buildGetUpdatedDataRequest(this.localNodeAddress, 1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); + Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); + } + + // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates + // correct GetDataRequestMessage with both sets of keys. + @Test + public void buildPreliminaryGetDataRequest_FilledP2PDataStore() throws NoSuchAlgorithmException { + PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); + ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); + + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd1, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd2, this.localNodeAddress, false, false, false, false); + + this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + + PreliminaryGetDataRequest getDataRequest = this.testState.mockedStorage.buildPreliminaryGetDataRequest(1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); + } + + // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates + // correct GetDataRequestMessage with both sets of keys. + @Test + public void requestData_FilledP2PDataStore_GetUpdatedDataRequest() throws NoSuchAlgorithmException { + PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); + ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); + + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd1, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload( + toAdd2, this.localNodeAddress, false, false, false, false); + + this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + + GetUpdatedDataRequest getDataRequest = + this.testState.mockedStorage.buildGetUpdatedDataRequest(this.localNodeAddress, 1); + + Assert.assertEquals(getDataRequest.getNonce(), 1); + Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); + Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); + Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), + P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); + } +} From daffe6dc382f68b56ac89ebf20a9b0d1104ac5ce Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 20 Nov 2019 17:47:56 -0800 Subject: [PATCH 04/51] [TESTS] Add tests of GetDataRequestHandler Add some basic sanity tests prior to the refactor to help catch issues. --- .../getdata/GetDataRequestHandlerTest.java | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java new file mode 100644 index 00000000000..10a7555e785 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java @@ -0,0 +1,199 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.peers.getdata; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.TestState; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.Proto; +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; + +import com.google.common.util.concurrent.SettableFuture; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.*; + +public class GetDataRequestHandlerTest { + private TestState testState; + + @Mock + NetworkNode networkNode; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); + + // Set up basic capabilities to ensure message contains it. Ensure it is unique from other tests + // so we catch mismatch bugs. + Capabilities.app.addAll(Capability.DAO_FULL_NODE); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: Construct a request that requires excluding duplicates and adding missing entrys for + // PersistableNetworkPayloads and ProtectedStorageEntrys to verify the correct keys are added to the response. + @Test + public void handle_PreliminaryGetDataRequestTest() throws NoSuchAlgorithmException { + + // Construct a seed node w/ 2 PersistableNetworkPayloads and 2 ProtectedStorageEntrys and a PreliminaryGetDataRequest + // that only contains 1 known PersistableNetworkPayload and 1 known ProtectedStorageEntry as well as 2 unknowns + PersistableNetworkPayload pnp_onSeedNodeNotInRequest = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload pnp_onSeedNodeAndInRequest = new PersistableNetworkPayloadStub(new byte[] { 2 }); + PersistableNetworkPayload pnp_onlyInRequest = new PersistableNetworkPayloadStub(new byte[] { 3 }); + ProtectedStorageEntry pse_onSeedNodeNotInRequest = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry pse_onSeedNodeAndInRequest = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry pse_onlyInRequest = getProtectedStorageEntryForAdd(); + + this.testState.getMockedStorage().addPersistableNetworkPayload( + pnp_onSeedNodeNotInRequest, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addPersistableNetworkPayload( + pnp_onSeedNodeAndInRequest, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeNotInRequest, this.localNodeAddress, null, false); + this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeAndInRequest, this.localNodeAddress, null, false); + + SettableFuture sendFuture = mock(SettableFuture.class); + final ArgumentCaptor captor = ArgumentCaptor.forClass(GetDataResponse.class); + when(this.networkNode.sendMessage(any(Connection.class), captor.capture())).thenReturn(sendFuture); + + PreliminaryGetDataRequest getDataRequest = mock(PreliminaryGetDataRequest.class); + HashSet knownKeysSet = new HashSet<>(Arrays.asList( + pnp_onSeedNodeAndInRequest.getHash(), + pnp_onlyInRequest.getHash(), + P2PDataStorage.get32ByteHash(pse_onSeedNodeAndInRequest.getProtectedStoragePayload()), + P2PDataStorage.get32ByteHash(pse_onlyInRequest.getProtectedStoragePayload()))); + when(getDataRequest.getNonce()).thenReturn(1); + when(getDataRequest.getExcludedKeys()).thenReturn(knownKeysSet); + + Connection connection = mock(Connection.class); + when(connection.noCapabilityRequiredOrCapabilityIsSupported(any(Proto.class))).thenReturn(true); + + GetDataRequestHandler handler = + new GetDataRequestHandler(this.networkNode, this.testState.getMockedStorage(), null); + handler.handle(getDataRequest, connection); + + // Verify the request node is sent back only the 2 missing payloads + GetDataResponse getDataResponse = captor.getValue(); + Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); + Assert.assertFalse(getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getPersistableNetworkPayloadSet(), new HashSet<>(Collections.singletonList(pnp_onSeedNodeNotInRequest))); + Assert.assertEquals(getDataResponse.getDataSet(), new HashSet<>(Collections.singletonList(pse_onSeedNodeNotInRequest))); + } + + // TESTCASE: Same as above, but with an GetUpdatedDataRequest + @Test + public void handle_GetUpdatedDataRequestTest() throws NoSuchAlgorithmException { + + // Construct a seed node w/ 2 PersistableNetworkPayloads and 2 ProtectedStorageEntrys and a PreliminaryGetDataRequest + // that only contains 1 known PersistableNetworkPayload and 1 known ProtectedStorageEntry as well as 2 unknowns + PersistableNetworkPayload pnp_onSeedNodeNotInRequest = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload pnp_onSeedNodeAndInRequest = new PersistableNetworkPayloadStub(new byte[] { 2 }); + PersistableNetworkPayload pnp_onlyInRequest = new PersistableNetworkPayloadStub(new byte[] { 3 }); + ProtectedStorageEntry pse_onSeedNodeNotInRequest = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry pse_onSeedNodeAndInRequest = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry pse_onlyInRequest = getProtectedStorageEntryForAdd(); + + this.testState.getMockedStorage().addPersistableNetworkPayload( + pnp_onSeedNodeNotInRequest, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addPersistableNetworkPayload( + pnp_onSeedNodeAndInRequest, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeNotInRequest, this.localNodeAddress, null, false); + this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeAndInRequest, this.localNodeAddress, null, false); + + SettableFuture sendFuture = mock(SettableFuture.class); + final ArgumentCaptor captor = ArgumentCaptor.forClass(GetDataResponse.class); + when(this.networkNode.sendMessage(any(Connection.class), captor.capture())).thenReturn(sendFuture); + + GetUpdatedDataRequest getDataRequest = mock(GetUpdatedDataRequest.class); + HashSet knownKeysSet = new HashSet<>(Arrays.asList( + pnp_onSeedNodeAndInRequest.getHash(), + pnp_onlyInRequest.getHash(), + P2PDataStorage.get32ByteHash(pse_onSeedNodeAndInRequest.getProtectedStoragePayload()), + P2PDataStorage.get32ByteHash(pse_onlyInRequest.getProtectedStoragePayload()))); + when(getDataRequest.getNonce()).thenReturn(1); + when(getDataRequest.getExcludedKeys()).thenReturn(knownKeysSet); + + Connection connection = mock(Connection.class); + when(connection.noCapabilityRequiredOrCapabilityIsSupported(any(Proto.class))).thenReturn(true); + + GetDataRequestHandler handler = + new GetDataRequestHandler(this.networkNode, this.testState.getMockedStorage(), null); + handler.handle(getDataRequest, connection); + + // Verify the request node is sent back only the 2 missing payloads + GetDataResponse getDataResponse = captor.getValue(); + Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); + Assert.assertTrue(getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getPersistableNetworkPayloadSet(), new HashSet<>(Collections.singletonList(pnp_onSeedNodeNotInRequest))); + Assert.assertEquals(getDataResponse.getDataSet(), new HashSet<>(Collections.singletonList(pse_onSeedNodeNotInRequest))); + } +} From 944b3fffbc74ad3a30e4e65615ce38e529f791f9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 20 Nov 2019 17:55:41 -0800 Subject: [PATCH 05/51] [REFACTOR] Introduce buildGetDataResponse This is just a strict move of code to reduce errors. --- .../peers/getdata/GetDataRequestHandler.java | 93 +------------------ .../network/p2p/storage/P2PDataStorage.java | 91 ++++++++++++++++++ 2 files changed, 92 insertions(+), 92 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index 0972cf32c8c..c1aff5476a5 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -22,28 +22,16 @@ import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.getdata.messages.GetDataRequest; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; -import bisq.network.p2p.storage.payload.PersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.Timer; import bisq.common.UserThread; -import bisq.common.util.Utilities; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -51,7 +39,6 @@ @Slf4j public class GetDataRequestHandler { private static final long TIMEOUT = 90; - private static final int MAX_ENTRIES = 10000; /////////////////////////////////////////////////////////////////////////////////////////// @@ -93,10 +80,7 @@ public GetDataRequestHandler(NetworkNode networkNode, P2PDataStorage dataStorage public void handle(GetDataRequest getDataRequest, final Connection connection) { long ts = System.currentTimeMillis(); - GetDataResponse getDataResponse = new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connection), - getFilteredPersistableNetworkPayload(getDataRequest, connection), - getDataRequest.getNonce(), - getDataRequest instanceof GetUpdatedDataRequest); + GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(getDataRequest, connection); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions @@ -136,81 +120,6 @@ public void onFailure(@NotNull Throwable throwable) { log.info("handle GetDataRequest took {} ms", System.currentTimeMillis() - ts); } - private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, - Connection connection) { - Set tempLookupSet = new HashSet<>(); - String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() - .map(e -> "node address " + e.getFullAddress()) - .orElseGet(() -> "connection UID " + connection.getUid()); - - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); - Set result = dataStorage.getAppendOnlyDataStoreMap().entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSize.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .filter(connection::noCapabilityRequiredOrCapabilityIsSupported) - .filter(payload -> { - boolean notContained = tempLookupSet.add(new P2PDataStorage.ByteArray(payload.getHash())); - return notContained; - }) - .collect(Collectors.toSet()); - if (maxSize.get() <= 0) { - log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + - "entries to get delivered. We limited the entries for the response to {} entries", - connectionInfo, MAX_ENTRIES); - } - log.info("The getData request from peer with {} contains {} PersistableNetworkPayload entries ", - connectionInfo, result.size()); - return result; - } - - private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, - Connection connection) { - Set filteredDataSet = new HashSet<>(); - Set lookupSet = new HashSet<>(); - String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() - .map(e -> "node address " + e.getFullAddress()) - .orElseGet(() -> "connection UID " + connection.getUid()); - - AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - Set filteredSet = dataStorage.getMap().entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSize.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .collect(Collectors.toSet()); - if (maxSize.get() <= 0) { - log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " + - "entries to get delivered. We limited the entries for the response to {} entries", - connectionInfo, MAX_ENTRIES); - } - log.info("getFilteredProtectedStorageEntries " + filteredSet.size()); - - for (ProtectedStorageEntry protectedStorageEntry : filteredSet) { - final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); - boolean doAdd = false; - if (protectedStoragePayload instanceof CapabilityRequiringPayload) { - if (connection.getCapabilities().containsAll(((CapabilityRequiringPayload) protectedStoragePayload).getRequiredCapabilities())) - doAdd = true; - else - log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + - "storagePayload is: " + Utilities.toTruncatedString(protectedStoragePayload)); - } else { - doAdd = true; - } - if (doAdd) { - boolean notContained = lookupSet.add(protectedStoragePayload.hashCode()); - if (notContained) - filteredDataSet.add(protectedStorageEntry); - } - } - - log.info("The getData request from peer with {} contains {} ProtectedStorageEntry entries ", - connectionInfo, filteredDataSet.size()); - return filteredDataSet; - } - public void stop() { cleanup(); } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index a66557ccd3c..64a71cb65b8 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -25,6 +25,8 @@ import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.BroadcastHandler; import bisq.network.p2p.peers.Broadcaster; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.messages.AddDataMessage; @@ -34,6 +36,7 @@ import bisq.network.p2p.storage.messages.RefreshOfferMessage; import bisq.network.p2p.storage.messages.RemoveDataMessage; import bisq.network.p2p.storage.messages.RemoveMailboxDataMessage; +import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; import bisq.network.p2p.storage.payload.ExpirablePayload; import bisq.network.p2p.storage.payload.MailboxStoragePayload; @@ -90,6 +93,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; @@ -110,6 +114,8 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers @VisibleForTesting public static int CHECK_TTL_INTERVAL_SEC = 60; + private static final int MAX_ENTRIES = 10000; + private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; private final ProtectedDataStoreService protectedDataStoreService; @@ -219,6 +225,91 @@ private Set getKnownPayloadHashes() { return excludedKeys; } + /** + * Returns a GetDataResponse object that contains the Payloads known locally, but not remotely. + */ + public GetDataResponse buildGetDataResponse(GetDataRequest getDataRequest, Connection connection) { + return new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connection), + getFilteredPersistableNetworkPayload(getDataRequest, connection), + getDataRequest.getNonce(), + getDataRequest instanceof GetUpdatedDataRequest); + } + + private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, + Connection connection) { + Set tempLookupSet = new HashSet<>(); + String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() + .map(e -> "node address " + e.getFullAddress()) + .orElseGet(() -> "connection UID " + connection.getUid()); + + Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); + AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); + Set result = this.appendOnlyDataStoreService.getMap().entrySet().stream() + .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) + .filter(e -> maxSize.decrementAndGet() >= 0) + .map(Map.Entry::getValue) + .filter(connection::noCapabilityRequiredOrCapabilityIsSupported) + .filter(payload -> { + boolean notContained = tempLookupSet.add(new P2PDataStorage.ByteArray(payload.getHash())); + return notContained; + }) + .collect(Collectors.toSet()); + if (maxSize.get() <= 0) { + log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + + "entries to get delivered. We limited the entries for the response to {} entries", + connectionInfo, MAX_ENTRIES); + } + log.info("The getData request from peer with {} contains {} PersistableNetworkPayload entries ", + connectionInfo, result.size()); + return result; + } + + private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, + Connection connection) { + Set filteredDataSet = new HashSet<>(); + Set lookupSet = new HashSet<>(); + String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() + .map(e -> "node address " + e.getFullAddress()) + .orElseGet(() -> "connection UID " + connection.getUid()); + + AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); + Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); + Set filteredSet = this.map.entrySet().stream() + .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) + .filter(e -> maxSize.decrementAndGet() >= 0) + .map(Map.Entry::getValue) + .collect(Collectors.toSet()); + if (maxSize.get() <= 0) { + log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " + + "entries to get delivered. We limited the entries for the response to {} entries", + connectionInfo, MAX_ENTRIES); + } + log.info("getFilteredProtectedStorageEntries " + filteredSet.size()); + + for (ProtectedStorageEntry protectedStorageEntry : filteredSet) { + final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); + boolean doAdd = false; + if (protectedStoragePayload instanceof CapabilityRequiringPayload) { + if (connection.getCapabilities().containsAll(((CapabilityRequiringPayload) protectedStoragePayload).getRequiredCapabilities())) + doAdd = true; + else + log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + + "storagePayload is: " + Utilities.toTruncatedString(protectedStoragePayload)); + } else { + doAdd = true; + } + if (doAdd) { + boolean notContained = lookupSet.add(protectedStoragePayload.hashCode()); + if (notContained) + filteredDataSet.add(protectedStorageEntry); + } + } + + log.info("The getData request from peer with {} contains {} ProtectedStorageEntry entries ", + connectionInfo, filteredDataSet.size()); + return filteredDataSet; + } + /////////////////////////////////////////////////////////////////////////////////////////// // API From 8208f7883716f61a20bd34ca18554f80b2dd5377 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 07:40:51 -0800 Subject: [PATCH 06/51] [REFACTOR] Extract connectionInfo String --- .../p2p/peers/getdata/GetDataRequestHandler.java | 6 +++++- .../bisq/network/p2p/storage/P2PDataStorage.java | 14 +++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index c1aff5476a5..dc298541ff1 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -80,7 +80,11 @@ public GetDataRequestHandler(NetworkNode networkNode, P2PDataStorage dataStorage public void handle(GetDataRequest getDataRequest, final Connection connection) { long ts = System.currentTimeMillis(); - GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(getDataRequest, connection); + String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() + .map(e -> "node address " + e.getFullAddress()) + .orElseGet(() -> "connection UID " + connection.getUid()); + + GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(getDataRequest, connectionInfo, connection); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 64a71cb65b8..5facada9bd3 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -228,19 +228,17 @@ private Set getKnownPayloadHashes() { /** * Returns a GetDataResponse object that contains the Payloads known locally, but not remotely. */ - public GetDataResponse buildGetDataResponse(GetDataRequest getDataRequest, Connection connection) { - return new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connection), - getFilteredPersistableNetworkPayload(getDataRequest, connection), + public GetDataResponse buildGetDataResponse(GetDataRequest getDataRequest, String connectionInfo, Connection connection) { + return new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connectionInfo, connection), + getFilteredPersistableNetworkPayload(getDataRequest, connectionInfo, connection), getDataRequest.getNonce(), getDataRequest instanceof GetUpdatedDataRequest); } private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, + String connectionInfo, Connection connection) { Set tempLookupSet = new HashSet<>(); - String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() - .map(e -> "node address " + e.getFullAddress()) - .orElseGet(() -> "connection UID " + connection.getUid()); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); @@ -265,12 +263,10 @@ private Set getFilteredPersistableNetworkPayload(GetD } private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, + String connectionInfo, Connection connection) { Set filteredDataSet = new HashSet<>(); Set lookupSet = new HashSet<>(); - String connectionInfo = "connectionInfo" + connection.getPeersNodeAddressOptional() - .map(e -> "node address " + e.getFullAddress()) - .orElseGet(() -> "connection UID " + connection.getUid()); AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); From 54021551bf54e112202e379ad7dd2899aaac14e1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 07:47:32 -0800 Subject: [PATCH 07/51] [REFACTOR] Extract getDataResponse logging Changed the log to reference getDataResponse instead of getData. Now that we might truncate the response, it ins't true that this is exactly what the peer asked. --- .../network/p2p/peers/getdata/GetDataRequestHandler.java | 5 +++++ .../main/java/bisq/network/p2p/storage/P2PDataStorage.java | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index dc298541ff1..91516c0989e 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -86,6 +86,11 @@ public void handle(GetDataRequest getDataRequest, final Connection connection) { GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(getDataRequest, connectionInfo, connection); + log.info("The getDataResponse to peer with {} contains {} ProtectedStorageEntries and {} PersistableNetworkPayloads", + connectionInfo, + getDataResponse.getDataSet().size(), + getDataResponse.getPersistableNetworkPayloadSet().size()); + if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions String errorMessage = "A timeout occurred for getDataResponse " + diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 5facada9bd3..9ee971f5009 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -257,8 +257,6 @@ private Set getFilteredPersistableNetworkPayload(GetD "entries to get delivered. We limited the entries for the response to {} entries", connectionInfo, MAX_ENTRIES); } - log.info("The getData request from peer with {} contains {} PersistableNetworkPayload entries ", - connectionInfo, result.size()); return result; } @@ -280,7 +278,6 @@ private Set getFilteredProtectedStorageEntries(GetDataReq "entries to get delivered. We limited the entries for the response to {} entries", connectionInfo, MAX_ENTRIES); } - log.info("getFilteredProtectedStorageEntries " + filteredSet.size()); for (ProtectedStorageEntry protectedStorageEntry : filteredSet) { final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); @@ -300,9 +297,6 @@ private Set getFilteredProtectedStorageEntries(GetDataReq filteredDataSet.add(protectedStorageEntry); } } - - log.info("The getData request from peer with {} contains {} ProtectedStorageEntry entries ", - connectionInfo, filteredDataSet.size()); return filteredDataSet; } From a6e8868563a1487b19fa4740106ec7b835ace300 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 08:02:38 -0800 Subject: [PATCH 08/51] [REFACTOR] Extract truncation logging Move the logging that utilizes connection information into the request handler. Now, buildGetDataResponse just returns whether or not the list is truncated which will make it easier to test. --- .../peers/getdata/GetDataRequestHandler.java | 24 ++++++++++- .../network/p2p/storage/P2PDataStorage.java | 43 +++++++++++-------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index 91516c0989e..eb2b56a3f71 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -32,6 +32,8 @@ import com.google.common.util.concurrent.SettableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -40,6 +42,7 @@ public class GetDataRequestHandler { private static final long TIMEOUT = 90; + private static final int MAX_ENTRIES = 10000; /////////////////////////////////////////////////////////////////////////////////////////// // Listener @@ -84,7 +87,26 @@ public void handle(GetDataRequest getDataRequest, final Connection connection) { .map(e -> "node address " + e.getFullAddress()) .orElseGet(() -> "connection UID " + connection.getUid()); - GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(getDataRequest, connectionInfo, connection); + AtomicBoolean outPersistableNetworkPayloadOutputTruncated = new AtomicBoolean(false); + AtomicBoolean outProtectedStoragePayloadOutputTruncated = new AtomicBoolean(false); + GetDataResponse getDataResponse = dataStorage.buildGetDataResponse( + getDataRequest, + MAX_ENTRIES, + outPersistableNetworkPayloadOutputTruncated, + outProtectedStoragePayloadOutputTruncated, + connection); + + if (outPersistableNetworkPayloadOutputTruncated.get()) { + log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + + "entries to get delivered. We limited the entries for the response to {} entries", + connectionInfo, MAX_ENTRIES); + } + + if (outProtectedStoragePayloadOutputTruncated.get()) { + log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " + + "entries to get delivered. We limited the entries for the response to {} entries", + connectionInfo, MAX_ENTRIES); + } log.info("The getDataResponse to peer with {} contains {} ProtectedStorageEntries and {} PersistableNetworkPayloads", connectionInfo, diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 9ee971f5009..458e22442e2 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -93,6 +93,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -114,8 +115,6 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers @VisibleForTesting public static int CHECK_TTL_INTERVAL_SEC = 60; - private static final int MAX_ENTRIES = 10000; - private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; private final ProtectedDataStoreService protectedDataStoreService; @@ -228,20 +227,28 @@ private Set getKnownPayloadHashes() { /** * Returns a GetDataResponse object that contains the Payloads known locally, but not remotely. */ - public GetDataResponse buildGetDataResponse(GetDataRequest getDataRequest, String connectionInfo, Connection connection) { - return new GetDataResponse(getFilteredProtectedStorageEntries(getDataRequest, connectionInfo, connection), - getFilteredPersistableNetworkPayload(getDataRequest, connectionInfo, connection), + public GetDataResponse buildGetDataResponse( + GetDataRequest getDataRequest, + int maxEntriesPerType, + AtomicBoolean outPersistableNetworkPayloadOutputTruncated, + AtomicBoolean outProtectedStorageEntryOutputTruncated, + Connection connection) { + + return new GetDataResponse( + getFilteredProtectedStorageEntries(getDataRequest, maxEntriesPerType, outProtectedStorageEntryOutputTruncated, connection), + getFilteredPersistableNetworkPayload(getDataRequest, maxEntriesPerType, outPersistableNetworkPayloadOutputTruncated, connection), getDataRequest.getNonce(), getDataRequest instanceof GetUpdatedDataRequest); } private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, - String connectionInfo, + int maxEntries, + AtomicBoolean outPersistableNetworkPayloadOutputTruncated, Connection connection) { Set tempLookupSet = new HashSet<>(); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); + AtomicInteger maxSize = new AtomicInteger(maxEntries); Set result = this.appendOnlyDataStoreService.getMap().entrySet().stream() .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) .filter(e -> maxSize.decrementAndGet() >= 0) @@ -252,32 +259,30 @@ private Set getFilteredPersistableNetworkPayload(GetD return notContained; }) .collect(Collectors.toSet()); - if (maxSize.get() <= 0) { - log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + - "entries to get delivered. We limited the entries for the response to {} entries", - connectionInfo, MAX_ENTRIES); - } + + if (maxSize.get() <= 0) + outPersistableNetworkPayloadOutputTruncated.set(true); + return result; } private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, - String connectionInfo, + int maxEntries, + AtomicBoolean outProtectedStorageEntryOutputTruncated, Connection connection) { Set filteredDataSet = new HashSet<>(); Set lookupSet = new HashSet<>(); - AtomicInteger maxSize = new AtomicInteger(MAX_ENTRIES); + AtomicInteger maxSize = new AtomicInteger(maxEntries); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); Set filteredSet = this.map.entrySet().stream() .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) .filter(e -> maxSize.decrementAndGet() >= 0) .map(Map.Entry::getValue) .collect(Collectors.toSet()); - if (maxSize.get() <= 0) { - log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " + - "entries to get delivered. We limited the entries for the response to {} entries", - connectionInfo, MAX_ENTRIES); - } + + if (maxSize.get() <= 0) + outProtectedStorageEntryOutputTruncated.set(true); for (ProtectedStorageEntry protectedStorageEntry : filteredSet) { final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); From dafc76200a67bf1ec68aac005d726e88dd40bd43 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 08:17:23 -0800 Subject: [PATCH 09/51] [REFACTOR] Pass peerCapabilities into buildGetDataResponse Remove the dependence on the connection object by having the handler pass in the peer's capabilities. This now allows unit testing of buildGetDataResponse without any connection dependencies. --- .../peers/getdata/GetDataRequestHandler.java | 2 +- .../network/p2p/storage/P2PDataStorage.java | 32 +++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index eb2b56a3f71..bb2c4949f9f 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -94,7 +94,7 @@ public void handle(GetDataRequest getDataRequest, final Connection connection) { MAX_ENTRIES, outPersistableNetworkPayloadOutputTruncated, outProtectedStoragePayloadOutputTruncated, - connection); + connection.getCapabilities()); if (outPersistableNetworkPayloadOutputTruncated.get()) { log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " + diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 458e22442e2..db09ef94493 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -53,6 +53,7 @@ import bisq.common.Timer; import bisq.common.UserThread; +import bisq.common.app.Capabilities; import bisq.common.crypto.CryptoException; import bisq.common.crypto.Hash; import bisq.common.crypto.Sig; @@ -232,19 +233,36 @@ public GetDataResponse buildGetDataResponse( int maxEntriesPerType, AtomicBoolean outPersistableNetworkPayloadOutputTruncated, AtomicBoolean outProtectedStorageEntryOutputTruncated, - Connection connection) { + Capabilities peerCapabilities) { return new GetDataResponse( - getFilteredProtectedStorageEntries(getDataRequest, maxEntriesPerType, outProtectedStorageEntryOutputTruncated, connection), - getFilteredPersistableNetworkPayload(getDataRequest, maxEntriesPerType, outPersistableNetworkPayloadOutputTruncated, connection), + getFilteredProtectedStorageEntries(getDataRequest, maxEntriesPerType, outProtectedStorageEntryOutputTruncated, peerCapabilities), + getFilteredPersistableNetworkPayload(getDataRequest, maxEntriesPerType, outPersistableNetworkPayloadOutputTruncated, peerCapabilities), getDataRequest.getNonce(), getDataRequest instanceof GetUpdatedDataRequest); } + /** + * Returns true if a Payload should be transmit to a peer given the peer's supported capabilities. + */ + private boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, NetworkPayload payload) { + + // Sanity check to ensure this isn't used outside P2PDataStorage + if (!(payload instanceof ProtectedStoragePayload || payload instanceof PersistableNetworkPayload)) + return false; + + // If the payload doesn't have a required capability, we should transmit it + if (!(payload instanceof CapabilityRequiringPayload)) + return true; + + // Otherwise, only transmit the Payload if the peer supports all capabilities required by the payload + return peerCapabilities.containsAll(((CapabilityRequiringPayload) payload).getRequiredCapabilities()); + } + private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, int maxEntries, AtomicBoolean outPersistableNetworkPayloadOutputTruncated, - Connection connection) { + Capabilities peerCapabilities) { Set tempLookupSet = new HashSet<>(); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); @@ -253,7 +271,7 @@ private Set getFilteredPersistableNetworkPayload(GetD .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) .filter(e -> maxSize.decrementAndGet() >= 0) .map(Map.Entry::getValue) - .filter(connection::noCapabilityRequiredOrCapabilityIsSupported) + .filter(persistableNetworkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, persistableNetworkPayload)) .filter(payload -> { boolean notContained = tempLookupSet.add(new P2PDataStorage.ByteArray(payload.getHash())); return notContained; @@ -269,7 +287,7 @@ private Set getFilteredPersistableNetworkPayload(GetD private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, int maxEntries, AtomicBoolean outProtectedStorageEntryOutputTruncated, - Connection connection) { + Capabilities peerCapabilities) { Set filteredDataSet = new HashSet<>(); Set lookupSet = new HashSet<>(); @@ -288,7 +306,7 @@ private Set getFilteredProtectedStorageEntries(GetDataReq final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); boolean doAdd = false; if (protectedStoragePayload instanceof CapabilityRequiringPayload) { - if (connection.getCapabilities().containsAll(((CapabilityRequiringPayload) protectedStoragePayload).getRequiredCapabilities())) + if (shouldTransmitPayloadToPeer(peerCapabilities, protectedStoragePayload)) doAdd = true; else log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + From 5630b3575594d2eae9fe5b728b8cef9b5eb9a7c8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 10:08:31 -0800 Subject: [PATCH 10/51] [TESTS] Unit tests of buildGetDataResponse Write a full set of unit tests for buildGetDataResponse. This provides a safety net with additional refactoring work. --- ...2PDataStorageBuildGetDataResponseTest.java | 483 ++++++++++++++++++ 1 file changed, 483 insertions(+) create mode 100644 p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java new file mode 100644 index 00000000000..fd9dea92bfd --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java @@ -0,0 +1,483 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; +import bisq.common.crypto.Sig; + +import com.google.protobuf.Message; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + + + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class P2PDataStorageBuildGetDataResponseTest { + abstract static class P2PDataStorageBuildGetDataResponseTestBase { + // GIVEN null & non-null supportedCapabilities + private TestState testState; + + abstract GetDataRequest buildGetDataRequest(int nonce, Set knownKeys); + + @Mock + NetworkNode networkNode; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); + + // Set up basic capabilities to ensure message contains it + Capabilities.app.addAll(Capability.MEDIATION); + } + + static class RequiredCapabilitiesPNPStub extends PersistableNetworkPayloadStub + implements CapabilityRequiringPayload { + Capabilities capabilities; + + RequiredCapabilitiesPNPStub(Capabilities capabilities, byte[] hash) { + super(hash); + this.capabilities = capabilities; + } + + @Override + public Capabilities getRequiredCapabilities() { + return capabilities; + } + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + return getProtectedStorageEntryForAdd(null); + } + + private ProtectedStorageEntry getProtectedStorageEntryForAdd(Capabilities requiredCapabilities) + throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + // Payload stub + ProtectedStoragePayload protectedStoragePayload; + + if (requiredCapabilities == null) + protectedStoragePayload = mock(ProtectedStoragePayload.class); + else { + protectedStoragePayload = mock(ProtectedStoragePayload.class, + withSettings().extraInterfaces(CapabilityRequiringPayload.class)); + when(((CapabilityRequiringPayload) protectedStoragePayload).getRequiredCapabilities()) + .thenReturn(requiredCapabilities); + } + + Message messageMock = mock(Message.class); + when(messageMock.toByteArray()).thenReturn(Sig.getPublicKeyBytes(ownerKeys.getPublic())); + when(protectedStoragePayload.toProtoMessage()).thenReturn(messageMock); + + // Entry stub + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: Given a GetDataRequest w/ unknown PNP, nothing is sent back + @Test + public void buildGetDataResponse_unknownPNPDoNothing() { + PersistableNetworkPayload fromPeer = new PersistableNetworkPayloadStub(new byte[]{1}); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>(Collections.singletonList(fromPeer.getHash()))); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/ known PNP, nothing is sent back + @Test + public void buildGetDataResponse_knownPNPDoNothing() { + PersistableNetworkPayload fromPeerAndLocal = new PersistableNetworkPayloadStub(new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + fromPeerAndLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest( + 1, + new HashSet<>(Collections.singletonList(fromPeerAndLocal.getHash()))); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, send it back + // XXXBUGXXX: Truncation return has off-by-one error + @Test + public void buildGetDataResponse_unknownPNPSendBack() { + PersistableNetworkPayload onlyLocal = new PersistableNetworkPayloadStub(new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().contains(onlyLocal)); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, don't send more than truncation limit + @Test + public void buildGetDataResponse_unknownPNPSendBackTruncation() { + PersistableNetworkPayload onlyLocal1 = new PersistableNetworkPayloadStub(new byte[]{1}); + PersistableNetworkPayload onlyLocal2 = new PersistableNetworkPayloadStub(new byte[]{2}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal1, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal2, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertTrue(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertEquals(1, getDataResponse.getPersistableNetworkPayloadSet().size()); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().contains(onlyLocal1)); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, but missing required capabilities, nothing is sent back + @Test + public void buildGetDataResponse_unknownPNPCapabilitiesMismatchDontSendBack() { + PersistableNetworkPayload onlyLocal = + new RequiredCapabilitiesPNPStub(new Capabilities(Collections.singletonList(Capability.MEDIATION)), + new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP that requires capabilities (and they match) send it back + @Test + public void buildGetDataResponse_unknownPNPCapabilitiesMatch() { + PersistableNetworkPayload onlyLocal = + new RequiredCapabilitiesPNPStub(new Capabilities(Collections.singletonList(Capability.MEDIATION)), + new byte[]{1}); + + this.testState.mockedStorage.addPersistableNetworkPayload( + onlyLocal, this.localNodeAddress, false, false, false, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(Collections.singletonList(Capability.MEDIATION)); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().contains(onlyLocal)); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/ unknown PSE, nothing is sent back + @Test + public void buildGetDataResponse_unknownPSEDoNothing() throws NoSuchAlgorithmException { + ProtectedStorageEntry fromPeer = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, + new HashSet<>(Collections.singletonList( + P2PDataStorage.get32ByteHash(fromPeer.getProtectedStoragePayload())))); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/ known PSE, nothing is sent back + @Test + public void buildGetDataResponse_knownPSEDoNothing() throws NoSuchAlgorithmException { + ProtectedStorageEntry fromPeerAndLocal = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, + new HashSet<>(Collections.singletonList( + P2PDataStorage.get32ByteHash(fromPeerAndLocal.getProtectedStoragePayload())))); + + this.testState.mockedStorage.addProtectedStorageEntry( + fromPeerAndLocal, this.localNodeAddress, null, false); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PSE, send it back + // XXXBUGXXX: Truncation return has off-by-one error + @Test + public void buildGetDataResponse_unknownPSESendBack() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal, this.localNodeAddress, null, false); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().contains(onlyLocal)); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, don't send more than truncation limit + @Test + public void buildGetDataResponse_unknownPSESendBackTruncation() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal1 = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry onlyLocal2 = getProtectedStorageEntryForAdd(); + + GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal1, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal2, this.localNodeAddress, null, false); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertTrue(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertEquals(1, getDataResponse.getDataSet().size()); + Assert.assertTrue( + getDataResponse.getDataSet().contains(onlyLocal1) + || getDataResponse.getDataSet().contains(onlyLocal2)); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP, but missing required capabilities, nothing is sent back + @Test + public void buildGetDataResponse_unknownPSECapabilitiesMismatchDontSendBack() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal = + getProtectedStorageEntryForAdd(new Capabilities(Collections.singletonList(Capability.MEDIATION))); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal, this.localNodeAddress, null, false); + + GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().isEmpty()); + } + + // TESTCASE: Given a GetDataRequest w/o known PNP that requires capabilities (and they match) send it back + @Test + public void buildGetDataResponse_unknownPSECapabilitiesMatch() throws NoSuchAlgorithmException { + ProtectedStorageEntry onlyLocal = + getProtectedStorageEntryForAdd(new Capabilities(Collections.singletonList(Capability.MEDIATION))); + + this.testState.mockedStorage.addProtectedStorageEntry( + onlyLocal, this.localNodeAddress, null, false); + + GetDataRequest getDataRequest = + this.buildGetDataRequest(1, new HashSet<>()); + + AtomicBoolean outPNPTruncated = new AtomicBoolean(false); + AtomicBoolean outPSETruncated = new AtomicBoolean(false); + Capabilities peerCapabilities = new Capabilities(Collections.singletonList(Capability.MEDIATION)); + GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( + getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + + Assert.assertFalse(outPNPTruncated.get()); + Assert.assertFalse(outPSETruncated.get()); + Assert.assertEquals(1, getDataResponse.getRequestNonce()); + Assert.assertEquals(getDataRequest instanceof GetUpdatedDataRequest, getDataResponse.isGetUpdatedDataResponse()); + Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); + Assert.assertTrue(getDataResponse.getPersistableNetworkPayloadSet().isEmpty()); + Assert.assertTrue(getDataResponse.getDataSet().contains(onlyLocal)); + } + } + + public static class P2PDataStorageBuildGetDataResponseTestPreliminary extends P2PDataStorageBuildGetDataResponseTestBase { + + @Override + GetDataRequest buildGetDataRequest(int nonce, Set knownKeys) { + return new PreliminaryGetDataRequest(nonce, knownKeys); + } + } + + public static class P2PDataStorageBuildGetDataResponseTestUpdated extends P2PDataStorageBuildGetDataResponseTestBase { + + @Override + GetDataRequest buildGetDataRequest(int nonce, Set knownKeys) { + return new GetUpdatedDataRequest(new NodeAddress("peer", 10), nonce, knownKeys); + } + } +} From caf946dfe009c3500aef90cf61f1f4b6dc64c70a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 10:13:20 -0800 Subject: [PATCH 11/51] Remove redundant HashSet lookups in filter functions The appendOnlyDataStoreService and map already have unique keys that are based on the hash of the payload. This would catch instances where: PersistableNetworkPayload - None: The key is based on ByteArray(payload.getHash()) which is the same as this check. ProtectedStorageEntry - Cases where multiple PSEs contain payloads that have equivalent hashCode(), but different data.toProtoMessage().toByteArray(). I don't think it is a good idea to keep 2 "unique" methods on payloads. This is likely left over from a time when Payload hashCode() needed to be different than the hash of the payload. --- .../java/bisq/network/p2p/storage/P2PDataStorage.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index db09ef94493..091d56788d8 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -263,8 +263,6 @@ private Set getFilteredPersistableNetworkPayload(GetD int maxEntries, AtomicBoolean outPersistableNetworkPayloadOutputTruncated, Capabilities peerCapabilities) { - Set tempLookupSet = new HashSet<>(); - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); AtomicInteger maxSize = new AtomicInteger(maxEntries); Set result = this.appendOnlyDataStoreService.getMap().entrySet().stream() @@ -272,10 +270,6 @@ private Set getFilteredPersistableNetworkPayload(GetD .filter(e -> maxSize.decrementAndGet() >= 0) .map(Map.Entry::getValue) .filter(persistableNetworkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, persistableNetworkPayload)) - .filter(payload -> { - boolean notContained = tempLookupSet.add(new P2PDataStorage.ByteArray(payload.getHash())); - return notContained; - }) .collect(Collectors.toSet()); if (maxSize.get() <= 0) @@ -289,7 +283,6 @@ private Set getFilteredProtectedStorageEntries(GetDataReq AtomicBoolean outProtectedStorageEntryOutputTruncated, Capabilities peerCapabilities) { Set filteredDataSet = new HashSet<>(); - Set lookupSet = new HashSet<>(); AtomicInteger maxSize = new AtomicInteger(maxEntries); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); @@ -315,9 +308,7 @@ private Set getFilteredProtectedStorageEntries(GetDataReq doAdd = true; } if (doAdd) { - boolean notContained = lookupSet.add(protectedStoragePayload.hashCode()); - if (notContained) - filteredDataSet.add(protectedStorageEntry); + filteredDataSet.add(protectedStorageEntry); } } return filteredDataSet; From 703a9a0dddba77835d105c49b246eb18e58c71e3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 10:17:01 -0800 Subject: [PATCH 12/51] [REFACTOR] Move required capabilities log Move the logging function to the common capabilities check so it can run on both ProtectedStoragePayload and PersistableNetworkPayload objects --- .../bisq/network/p2p/storage/P2PDataStorage.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 091d56788d8..063fd83ab11 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -256,7 +256,14 @@ private boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, Netwo return true; // Otherwise, only transmit the Payload if the peer supports all capabilities required by the payload - return peerCapabilities.containsAll(((CapabilityRequiringPayload) payload).getRequiredCapabilities()); + boolean shouldTransmit = peerCapabilities.containsAll(((CapabilityRequiringPayload) payload).getRequiredCapabilities()); + + if (!shouldTransmit) { + log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + + "storagePayload is: " + Utilities.toTruncatedString(payload)); + } + + return shouldTransmit; } private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, @@ -301,9 +308,6 @@ private Set getFilteredProtectedStorageEntries(GetDataReq if (protectedStoragePayload instanceof CapabilityRequiringPayload) { if (shouldTransmitPayloadToPeer(peerCapabilities, protectedStoragePayload)) doAdd = true; - else - log.debug("We do not send the message to the peer because they do not support the required capability for that message type.\n" + - "storagePayload is: " + Utilities.toTruncatedString(protectedStoragePayload)); } else { doAdd = true; } From 3aaf8a285e61a9842e2127b669182c11f8d4317e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 10:21:37 -0800 Subject: [PATCH 13/51] [REFACTOR] Inline capability check for ProtectedStorageEntries Move the capability check inside the stream operation. This should improve performance slightly, but more importantly it makes the two filter functions almost identical so they can be combined. --- .../network/p2p/storage/P2PDataStorage.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 063fd83ab11..4e2cceca806 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -289,33 +289,19 @@ private Set getFilteredProtectedStorageEntries(GetDataReq int maxEntries, AtomicBoolean outProtectedStorageEntryOutputTruncated, Capabilities peerCapabilities) { - Set filteredDataSet = new HashSet<>(); - AtomicInteger maxSize = new AtomicInteger(maxEntries); Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); Set filteredSet = this.map.entrySet().stream() .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) .filter(e -> maxSize.decrementAndGet() >= 0) .map(Map.Entry::getValue) + .filter(protectedStorageEntry -> shouldTransmitPayloadToPeer(peerCapabilities, protectedStorageEntry.getProtectedStoragePayload())) .collect(Collectors.toSet()); if (maxSize.get() <= 0) outProtectedStorageEntryOutputTruncated.set(true); - for (ProtectedStorageEntry protectedStorageEntry : filteredSet) { - final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); - boolean doAdd = false; - if (protectedStoragePayload instanceof CapabilityRequiringPayload) { - if (shouldTransmitPayloadToPeer(peerCapabilities, protectedStoragePayload)) - doAdd = true; - } else { - doAdd = true; - } - if (doAdd) { - filteredDataSet.add(protectedStorageEntry); - } - } - return filteredDataSet; + return filteredSet; } From 4c5d8184b7761ea177cc52e2f60160178336073f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 10:26:30 -0800 Subject: [PATCH 14/51] [REFACTOR] Inline filtering functions Removes unnecessary calculations converting Set into Set and allows additional deduplication of stream operations. --- .../network/p2p/storage/P2PDataStorage.java | 66 +++++++------------ 1 file changed, 25 insertions(+), 41 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 4e2cceca806..64296ffd1b2 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -235,9 +235,32 @@ public GetDataResponse buildGetDataResponse( AtomicBoolean outProtectedStorageEntryOutputTruncated, Capabilities peerCapabilities) { + Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); + AtomicInteger maxSizePersistableNetworkPayloads = new AtomicInteger(maxEntriesPerType); + Set filteredPersistableNetworkPayloads = this.appendOnlyDataStoreService.getMap().entrySet().stream() + .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) + .filter(e -> maxSizePersistableNetworkPayloads.decrementAndGet() >= 0) + .map(Map.Entry::getValue) + .filter(persistableNetworkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, persistableNetworkPayload)) + .collect(Collectors.toSet()); + + if (maxSizePersistableNetworkPayloads.get() <= 0) + outPersistableNetworkPayloadOutputTruncated.set(true); + + AtomicInteger maxSizeProtectedStorageEntries = new AtomicInteger(maxEntriesPerType); + Set filteredProtectedStorageEntries = this.map.entrySet().stream() + .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) + .filter(e -> maxSizeProtectedStorageEntries.decrementAndGet() >= 0) + .map(Map.Entry::getValue) + .filter(protectedStorageEntry -> shouldTransmitPayloadToPeer(peerCapabilities, protectedStorageEntry.getProtectedStoragePayload())) + .collect(Collectors.toSet()); + + if (maxSizeProtectedStorageEntries.get() <= 0) + outProtectedStorageEntryOutputTruncated.set(true); + return new GetDataResponse( - getFilteredProtectedStorageEntries(getDataRequest, maxEntriesPerType, outProtectedStorageEntryOutputTruncated, peerCapabilities), - getFilteredPersistableNetworkPayload(getDataRequest, maxEntriesPerType, outPersistableNetworkPayloadOutputTruncated, peerCapabilities), + filteredProtectedStorageEntries, + filteredPersistableNetworkPayloads, getDataRequest.getNonce(), getDataRequest instanceof GetUpdatedDataRequest); } @@ -266,45 +289,6 @@ private boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, Netwo return shouldTransmit; } - private Set getFilteredPersistableNetworkPayload(GetDataRequest getDataRequest, - int maxEntries, - AtomicBoolean outPersistableNetworkPayloadOutputTruncated, - Capabilities peerCapabilities) { - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - AtomicInteger maxSize = new AtomicInteger(maxEntries); - Set result = this.appendOnlyDataStoreService.getMap().entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSize.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .filter(persistableNetworkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, persistableNetworkPayload)) - .collect(Collectors.toSet()); - - if (maxSize.get() <= 0) - outPersistableNetworkPayloadOutputTruncated.set(true); - - return result; - } - - private Set getFilteredProtectedStorageEntries(GetDataRequest getDataRequest, - int maxEntries, - AtomicBoolean outProtectedStorageEntryOutputTruncated, - Capabilities peerCapabilities) { - AtomicInteger maxSize = new AtomicInteger(maxEntries); - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - Set filteredSet = this.map.entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSize.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .filter(protectedStorageEntry -> shouldTransmitPayloadToPeer(peerCapabilities, protectedStorageEntry.getProtectedStoragePayload())) - .collect(Collectors.toSet()); - - if (maxSize.get() <= 0) - outProtectedStorageEntryOutputTruncated.set(true); - - return filteredSet; - } - - /////////////////////////////////////////////////////////////////////////////////////////// // API /////////////////////////////////////////////////////////////////////////////////////////// From e7673407f16b95363047f541682546638721ffa0 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 11:32:22 -0800 Subject: [PATCH 15/51] [REFACTOR] Remove duplication in filtering functions Introduce a generic function that can be used to filter Map or Map. Used to deduplicate the GetData code paths and ensure the logic is the same between the two payload types. --- .../network/p2p/storage/P2PDataStorage.java | 73 +++++++++++++------ 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 64296ffd1b2..2518e945d86 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -96,6 +96,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; @@ -225,6 +226,34 @@ private Set getKnownPayloadHashes() { return excludedKeys; } + /** + * Generic function that can be used to filter a Map + * by a given set of keys and peer capabilities. + */ + static private Set filterKnownHashes( + Map toFilter, + Function objToPayload, + Set knownHashes, + Capabilities peerCapabilities, + int maxEntries, + AtomicBoolean outTruncated) { + + AtomicInteger limit = new AtomicInteger(maxEntries); + + Set filteredResults = toFilter.entrySet().stream() + .filter(e -> !knownHashes.contains(e.getKey())) + .filter(e -> limit.decrementAndGet() >= 0) + .map(Map.Entry::getValue) + .filter(networkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, + objToPayload.apply(networkPayload))) + .collect(Collectors.toSet()); + + if (limit.get() <= 0) + outTruncated.set(true); + + return filteredResults; + } + /** * Returns a GetDataResponse object that contains the Payloads known locally, but not remotely. */ @@ -235,28 +264,26 @@ public GetDataResponse buildGetDataResponse( AtomicBoolean outProtectedStorageEntryOutputTruncated, Capabilities peerCapabilities) { - Set excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); - AtomicInteger maxSizePersistableNetworkPayloads = new AtomicInteger(maxEntriesPerType); - Set filteredPersistableNetworkPayloads = this.appendOnlyDataStoreService.getMap().entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSizePersistableNetworkPayloads.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .filter(persistableNetworkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, persistableNetworkPayload)) - .collect(Collectors.toSet()); - - if (maxSizePersistableNetworkPayloads.get() <= 0) - outPersistableNetworkPayloadOutputTruncated.set(true); - - AtomicInteger maxSizeProtectedStorageEntries = new AtomicInteger(maxEntriesPerType); - Set filteredProtectedStorageEntries = this.map.entrySet().stream() - .filter(e -> !excludedKeysAsByteArray.contains(e.getKey())) - .filter(e -> maxSizeProtectedStorageEntries.decrementAndGet() >= 0) - .map(Map.Entry::getValue) - .filter(protectedStorageEntry -> shouldTransmitPayloadToPeer(peerCapabilities, protectedStorageEntry.getProtectedStoragePayload())) - .collect(Collectors.toSet()); - - if (maxSizeProtectedStorageEntries.get() <= 0) - outProtectedStorageEntryOutputTruncated.set(true); + Set excludedKeysAsByteArray = + P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys()); + + Set filteredPersistableNetworkPayloads = + filterKnownHashes( + this.appendOnlyDataStoreService.getMap(), + Function.identity(), + excludedKeysAsByteArray, + peerCapabilities, + maxEntriesPerType, + outPersistableNetworkPayloadOutputTruncated); + + Set filteredProtectedStorageEntries = + filterKnownHashes( + this.map, + ProtectedStorageEntry::getProtectedStoragePayload, + excludedKeysAsByteArray, + peerCapabilities, + maxEntriesPerType, + outProtectedStorageEntryOutputTruncated); return new GetDataResponse( filteredProtectedStorageEntries, @@ -268,7 +295,7 @@ public GetDataResponse buildGetDataResponse( /** * Returns true if a Payload should be transmit to a peer given the peer's supported capabilities. */ - private boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, NetworkPayload payload) { + private static boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities, NetworkPayload payload) { // Sanity check to ensure this isn't used outside P2PDataStorage if (!(payload instanceof ProtectedStoragePayload || payload instanceof PersistableNetworkPayload)) From 00128d912df9b833ab715a4b0a85750c7603b226 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 11:37:55 -0800 Subject: [PATCH 16/51] [BUGFIX] Fix off-by-one in truncation logic Now, the truncation is only triggered if more than MAX_ENTRIES could have been returned. --- .../main/java/bisq/network/p2p/storage/P2PDataStorage.java | 2 +- .../p2p/storage/P2PDataStorageBuildGetDataResponseTest.java | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 2518e945d86..58f71b22c4c 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -248,7 +248,7 @@ static private Set filterKnownHashes( objToPayload.apply(networkPayload))) .collect(Collectors.toSet()); - if (limit.get() <= 0) + if (limit.get() < 0) outTruncated.set(true); return filteredResults; diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java index fd9dea92bfd..173d3358041 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java @@ -189,7 +189,6 @@ public void buildGetDataResponse_knownPNPDoNothing() { } // TESTCASE: Given a GetDataRequest w/o known PNP, send it back - // XXXBUGXXX: Truncation return has off-by-one error @Test public void buildGetDataResponse_unknownPNPSendBack() { PersistableNetworkPayload onlyLocal = new PersistableNetworkPayloadStub(new byte[]{1}); @@ -204,7 +203,7 @@ public void buildGetDataResponse_unknownPNPSendBack() { AtomicBoolean outPSETruncated = new AtomicBoolean(false); Capabilities peerCapabilities = new Capabilities(); GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( - getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); Assert.assertFalse(outPNPTruncated.get()); Assert.assertFalse(outPSETruncated.get()); @@ -355,7 +354,6 @@ public void buildGetDataResponse_knownPSEDoNothing() throws NoSuchAlgorithmExcep } // TESTCASE: Given a GetDataRequest w/o known PSE, send it back - // XXXBUGXXX: Truncation return has off-by-one error @Test public void buildGetDataResponse_unknownPSESendBack() throws NoSuchAlgorithmException { ProtectedStorageEntry onlyLocal = getProtectedStorageEntryForAdd(); @@ -369,7 +367,7 @@ public void buildGetDataResponse_unknownPSESendBack() throws NoSuchAlgorithmExce AtomicBoolean outPSETruncated = new AtomicBoolean(false); Capabilities peerCapabilities = new Capabilities(); GetDataResponse getDataResponse = this.testState.mockedStorage.buildGetDataResponse( - getDataRequest, 2, outPNPTruncated, outPSETruncated, peerCapabilities); + getDataRequest, 1, outPNPTruncated, outPSETruncated, peerCapabilities); Assert.assertFalse(outPNPTruncated.get()); Assert.assertFalse(outPSETruncated.get()); From c7bce9e999de2603beacf4ae99a73782814117d8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 13:16:00 -0800 Subject: [PATCH 17/51] [TESTS] Add test of RequestDataHandler::onMessage Add heavy-handed test that exercises the logic to use as a safeguard for refactoring. --- .../getdata/GetDataRequestOnMessageTest.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java new file mode 100644 index 00000000000..2ac1c25b884 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java @@ -0,0 +1,160 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.peers.getdata; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.PeerManager; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.storage.TestState; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; + +import com.google.common.util.concurrent.SettableFuture; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class GetDataRequestOnMessageTest { + private TestState testState; + + @Mock + NetworkNode networkNode; + + private NodeAddress localNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + + this.localNodeAddress = new NodeAddress("localhost", 8080); + when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); + + // Set up basic capabilities to ensure message contains it. Ensure it is unique from other tests + // so we catch mismatch bugs. + Capabilities.app.addAll(Capability.DAO_FULL_NODE); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayloadStub + implements LazyProcessedPayload { + + LazyPersistableNetworkPayloadStub(byte[] hash) { + super(hash); + } + } + + // TESTCASE: GetDataResponse processing. This large tests includes all interesting variations of state + @Test + public void onMessage_GetDataResponseTest() throws NoSuchAlgorithmException { + + PersistableNetworkPayload pnp_onLocalNodeOnly = new PersistableNetworkPayloadStub(new byte[] { 1 }); + PersistableNetworkPayload pnp_inRequestOnly = new LazyPersistableNetworkPayloadStub(new byte[]{2}); + PersistableNetworkPayload pnp_onLocalNodeAndRequest = new PersistableNetworkPayloadStub(new byte[] { 3 }); + ProtectedStorageEntry pse_onLocalNodeOnly = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry pse_inRequestOnly = getProtectedStorageEntryForAdd(); + ProtectedStorageEntry pse_onLocalNodeAndRequest = getProtectedStorageEntryForAdd(); + + this.testState.getMockedStorage().addPersistableNetworkPayload( + pnp_onLocalNodeOnly, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addPersistableNetworkPayload( + pnp_onLocalNodeAndRequest, this.localNodeAddress, false, false, false, false); + this.testState.getMockedStorage().addProtectedStorageEntry(pse_onLocalNodeOnly, this.localNodeAddress, null, false); + this.testState.getMockedStorage().addProtectedStorageEntry(pse_onLocalNodeAndRequest, this.localNodeAddress, null, false); + + RequestDataHandler handler = new RequestDataHandler( + this.networkNode, + this.testState.getMockedStorage(), + mock(PeerManager.class), + mock(RequestDataHandler.Listener.class) + ); + + GetDataResponse getDataResponse = new GetDataResponse( + new HashSet<>(Arrays.asList(pse_inRequestOnly, pse_onLocalNodeAndRequest)), + new HashSet<>(Arrays.asList(pnp_inRequestOnly, pnp_onLocalNodeAndRequest)), + handler.nonce, false); + + NodeAddress peersNodeAddress = new NodeAddress("peer", 10); + + // Make a request with the sole reason to set the peersNodeAddress + SettableFuture sendFuture = mock(SettableFuture.class); + when(networkNode.sendMessage(any(NodeAddress.class), any(GetDataRequest.class))).thenReturn(sendFuture); + handler.requestData(peersNodeAddress, true); + + Connection connection = mock(Connection.class); + when(connection.getPeersNodeAddressOptional()).thenReturn(Optional.of(peersNodeAddress)); + handler.onMessage(getDataResponse, connection); + + Assert.assertEquals(3, this.testState.getMockedStorage().getMap().size()); + Assert.assertTrue(this.testState.getMockedStorage().getAppendOnlyDataStoreMap().containsValue(pnp_onLocalNodeOnly)); + Assert.assertTrue(this.testState.getMockedStorage().getAppendOnlyDataStoreMap().containsValue(pnp_inRequestOnly)); + Assert.assertTrue(this.testState.getMockedStorage().getAppendOnlyDataStoreMap().containsValue(pnp_onLocalNodeAndRequest)); + + Assert.assertEquals(3, this.testState.getMockedStorage().getMap().size()); + Assert.assertTrue(this.testState.getMockedStorage().getMap().containsValue(pse_onLocalNodeOnly)); + Assert.assertTrue(this.testState.getMockedStorage().getMap().containsValue(pse_inRequestOnly)); + Assert.assertTrue(this.testState.getMockedStorage().getMap().containsValue(pse_onLocalNodeAndRequest)); + } +} From 873271c5ceb9664463adfa79b7439830a12608d8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 13:30:42 -0800 Subject: [PATCH 18/51] [REFACTOR] Introduce processGetDataResponse Just a code move for now. --- .../p2p/peers/getdata/RequestDataHandler.java | 52 +----------------- .../network/p2p/storage/P2PDataStorage.java | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index 10b1b7ae905..01c0f00361a 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -25,10 +25,7 @@ import bisq.network.p2p.peers.PeerManager; import bisq.network.p2p.peers.getdata.messages.GetDataRequest; import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; -import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; @@ -48,7 +45,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; @@ -58,7 +54,6 @@ @Slf4j class RequestDataHandler implements MessageListener { private static final long TIMEOUT = 90; - private static boolean initialRequestApplied = false; private NodeAddress peersNodeAddress; /* @@ -193,7 +188,6 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { GetDataResponse getDataResponse = (GetDataResponse) networkEnvelope; final Set dataSet = getDataResponse.getDataSet(); Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); - logContents(networkEnvelope, dataSet, persistableNetworkPayloadSet); if (getDataResponse.getRequestNonce() == nonce) { @@ -204,50 +198,8 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { return; } - final NodeAddress sender = connection.getPeersNodeAddressOptional().get(); - - long ts2 = System.currentTimeMillis(); - AtomicInteger counter = new AtomicInteger(); - dataSet.forEach(e -> { - // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless - dataStorage.addProtectedStorageEntry(e, sender, null, false, false); - counter.getAndIncrement(); - - }); - log.info("Processing {} protectedStorageEntries took {} ms.", counter.get(), System.currentTimeMillis() - ts2); - - /* // engage the firstRequest logic only if we are a seed node. Normal clients get here twice at most. - if (!Capabilities.app.containsAll(Capability.SEED_NODE)) - firstRequest = true;*/ - - if (persistableNetworkPayloadSet != null /*&& firstRequest*/) { - ts2 = System.currentTimeMillis(); - persistableNetworkPayloadSet.forEach(e -> { - if (e instanceof LazyProcessedPayload) { - // We use an optimized method as many checks are not required in that case to avoid - // performance issues. - // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). - // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all - // trade stats stats and all account age witness data. - - // We only apply it once from first response - if (!initialRequestApplied) { - dataStorage.addPersistableNetworkPayloadFromInitialRequest(e); - - } - } else { - // We don't broadcast here as we are only connected to the seed node and would be pointless - dataStorage.addPersistableNetworkPayload(e, sender, false, - false, false, false); - } - }); - - // We set initialRequestApplied to true after the loop, otherwise we would only process 1 entry - initialRequestApplied = true; - - log.info("Processing {} persistableNetworkPayloads took {} ms.", - persistableNetworkPayloadSet.size(), System.currentTimeMillis() - ts2); - } + dataStorage.processGetDataResponse(getDataResponse, + connection.getPeersNodeAddressOptional().get()); cleanup(); listener.onComplete(); diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 58f71b22c4c..6a70f68ad26 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -39,6 +39,7 @@ import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; import bisq.network.p2p.storage.payload.ExpirablePayload; +import bisq.network.p2p.storage.payload.LazyProcessedPayload; import bisq.network.p2p.storage.payload.MailboxStoragePayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry; @@ -117,6 +118,8 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers @VisibleForTesting public static int CHECK_TTL_INTERVAL_SEC = 60; + private static boolean initialRequestApplied = false; + private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; private final ProtectedDataStoreService protectedDataStoreService; @@ -316,6 +319,58 @@ private static boolean shouldTransmitPayloadToPeer(Capabilities peerCapabilities return shouldTransmit; } + /** + * Processes a GetDataResponse message and updates internal state. Does not broadcast updates to the P2P network + * or domain listeners. + */ + public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress sender) { + final Set dataSet = getDataResponse.getDataSet(); + Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); + + long ts2 = System.currentTimeMillis(); + AtomicInteger counter = new AtomicInteger(); + dataSet.forEach(e -> { + // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless + addProtectedStorageEntry(e, sender, null, false, false); + counter.getAndIncrement(); + + }); + log.info("Processing {} protectedStorageEntries took {} ms.", counter.get(), System.currentTimeMillis() - ts2); + + /* // engage the firstRequest logic only if we are a seed node. Normal clients get here twice at most. + if (!Capabilities.app.containsAll(Capability.SEED_NODE)) + firstRequest = true;*/ + + if (persistableNetworkPayloadSet != null /*&& firstRequest*/) { + ts2 = System.currentTimeMillis(); + persistableNetworkPayloadSet.forEach(e -> { + if (e instanceof LazyProcessedPayload) { + // We use an optimized method as many checks are not required in that case to avoid + // performance issues. + // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). + // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all + // trade stats stats and all account age witness data. + + // We only apply it once from first response + if (!initialRequestApplied) { + addPersistableNetworkPayloadFromInitialRequest(e); + + } + } else { + // We don't broadcast here as we are only connected to the seed node and would be pointless + addPersistableNetworkPayload(e, sender, false, + false, false, false); + } + }); + + // We set initialRequestApplied to true after the loop, otherwise we would only process 1 entry + initialRequestApplied = true; + + log.info("Processing {} persistableNetworkPayloads took {} ms.", + persistableNetworkPayloadSet.size(), System.currentTimeMillis() - ts2); + } + } + /////////////////////////////////////////////////////////////////////////////////////////// // API /////////////////////////////////////////////////////////////////////////////////////////// From 690b9808b148d2a7c108fb37f3f15ac67c246847 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 14:41:39 -0800 Subject: [PATCH 19/51] [TESTS] Make verify() functions more flexible Now that we want to unit test the GetData path which has different behavior w.r.t. broadcasts, the tests need a way to verify that state was updated, but not broadcast during an add. This patch changes all verification function to take each state update explicitly so the tests can do the proper verification. --- .../storage/P2PDataStorageClientAPITest.java | 6 +- ...aStoragePersistableNetworkPayloadTest.java | 4 +- ...PDataStorageProtectedStorageEntryTest.java | 8 ++- .../bisq/network/p2p/storage/TestState.java | 62 +++++++++---------- 4 files changed, 44 insertions(+), 36 deletions(-) diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index 4e0cb6c895f..7126b61770a 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -73,7 +73,7 @@ public void getProtectedStorageEntry_NoExist() throws NoSuchAlgorithmException, SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } // TESTCASE: Adding an entry from the getProtectedStorageEntry API of an existing item correctly updates the item @@ -90,7 +90,7 @@ public void getProtectedStorageEntry() throws NoSuchAlgorithmException, CryptoEx protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } // TESTCASE: Adding an entry from the getProtectedStorageEntry API of an existing item (added from onMessage path) correctly updates the item @@ -110,7 +110,7 @@ public void getProtectedStorageEntry_FirstOnMessageSecondAPI() throws NoSuchAlgo protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } // TESTCASE: Updating an entry from the getRefreshTTLMessage API correctly errors if the item hasn't been seen diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java index 9213bcb0fbe..45c1ce17689 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java @@ -107,7 +107,9 @@ void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean testState.mockedStorage.onMessage(new AddPersistableNetworkPayloadMessage(persistableNetworkPayload), mockedConnection); } - this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, this.expectBroadcastOnStateChange(), this.expectedIsDataOwner()); + boolean expectedBroadcast = expectedStateChange && this.expectBroadcastOnStateChange(); + + this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedBroadcast, expectedBroadcast, this.expectedIsDataOwner()); } @Before diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java index 9f5bd539234..04eb291f309 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java @@ -197,7 +197,13 @@ void doProtectedStorageAddAndVerify(ProtectedStorageEntry protectedStorageEntry, if (!this.useMessageHandler) Assert.assertEquals(expectedReturnValue, addResult); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, expectedStateChange, this.expectIsDataOwner()); + if (expectedStateChange) { + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, true, true, this.expectIsDataOwner()); + } else{ + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, false, false, false, false, this.expectIsDataOwner()); + } } void doProtectedStorageRemoveAndVerify(ProtectedStorageEntry entry, diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index 88d465c5734..15e80a9c6ac 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -174,52 +174,55 @@ private void verifySequenceNumberMapWriteContains(P2PDataStorage.ByteArray paylo void verifyPersistableAdd(SavedTestState beforeState, PersistableNetworkPayload persistableNetworkPayload, - boolean expectedStateChange, - boolean expectedBroadcastAndListenersSignaled, + boolean expectedHashMapAndDataStoreUpdated, + boolean expectedListenersSignaled, + boolean expectedBroadcast, boolean expectedIsDataOwner) { P2PDataStorage.ByteArray hash = new P2PDataStorage.ByteArray(persistableNetworkPayload.getHash()); - if (expectedStateChange) { - // Payload is accessible from get() + if (expectedHashMapAndDataStoreUpdated) Assert.assertEquals(persistableNetworkPayload, this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - } else { - // On failure, just ensure the state remained the same as before the add - if (beforeState.persistableNetworkPayloadBeforeOp != null) - Assert.assertEquals(beforeState.persistableNetworkPayloadBeforeOp, this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - else - Assert.assertNull(this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - } - - if (expectedStateChange && expectedBroadcastAndListenersSignaled) { - // Broadcast Called - verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), any(NodeAddress.class), - eq(null), eq(expectedIsDataOwner)); + else + Assert.assertEquals(beforeState.persistableNetworkPayloadBeforeOp, this.mockedStorage.getAppendOnlyDataStoreMap().get(hash)); - // Verify the listeners were updated once + if (expectedListenersSignaled) verify(this.appendOnlyDataStoreListener).onAdded(persistableNetworkPayload); + else + verify(this.appendOnlyDataStoreListener, never()).onAdded(persistableNetworkPayload); - } else { + if (expectedBroadcast) + verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), any(NodeAddress.class), + eq(null), eq(expectedIsDataOwner)); + else verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); - - // Verify the listeners were never updated - verify(this.appendOnlyDataStoreListener, never()).onAdded(persistableNetworkPayload); - } } void verifyProtectedStorageAdd(SavedTestState beforeState, ProtectedStorageEntry protectedStorageEntry, - boolean expectedStateChange, + boolean expectedHashMapAndDataStoreUpdated, + boolean expectedListenersSignaled, + boolean expectedBroadcast, + boolean expectedSequenceNrMapWrite, boolean expectedIsDataOwner) { P2PDataStorage.ByteArray hashMapHash = P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()); - if (expectedStateChange) { + if (expectedHashMapAndDataStoreUpdated) { Assert.assertEquals(protectedStorageEntry, this.mockedStorage.getMap().get(hashMapHash)); if (protectedStorageEntry.getProtectedStoragePayload() instanceof PersistablePayload) Assert.assertEquals(protectedStorageEntry, this.protectedDataStoreService.getMap().get(hashMapHash)); + } else { + Assert.assertEquals(beforeState.protectedStorageEntryBeforeOp, this.mockedStorage.getMap().get(hashMapHash)); + Assert.assertEquals(beforeState.protectedStorageEntryBeforeOpDataStoreMap, this.protectedDataStoreService.getMap().get(hashMapHash)); + } + if (expectedListenersSignaled) { verify(this.hashMapChangedListener).onAdded(Collections.singletonList(protectedStorageEntry)); + } else { + verify(this.hashMapChangedListener, never()).onAdded(Collections.singletonList(protectedStorageEntry)); + } + if (expectedBroadcast) { final ArgumentCaptor captor = ArgumentCaptor.forClass(BroadcastMessage.class); verify(this.mockBroadcaster).broadcast(captor.capture(), any(NodeAddress.class), eq(null), eq(expectedIsDataOwner)); @@ -227,16 +230,13 @@ void verifyProtectedStorageAdd(SavedTestState beforeState, BroadcastMessage broadcastMessage = captor.getValue(); Assert.assertTrue(broadcastMessage instanceof AddDataMessage); Assert.assertEquals(protectedStorageEntry, ((AddDataMessage) broadcastMessage).getProtectedStorageEntry()); - - this.verifySequenceNumberMapWriteContains(P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()), protectedStorageEntry.getSequenceNumber()); } else { - Assert.assertEquals(beforeState.protectedStorageEntryBeforeOp, this.mockedStorage.getMap().get(hashMapHash)); - Assert.assertEquals(beforeState.protectedStorageEntryBeforeOpDataStoreMap, this.protectedDataStoreService.getMap().get(hashMapHash)); - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); + } - // Internal state didn't change... nothing should be notified - verify(this.hashMapChangedListener, never()).onAdded(Collections.singletonList(protectedStorageEntry)); + if (expectedSequenceNrMapWrite) { + this.verifySequenceNumberMapWriteContains(P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()), protectedStorageEntry.getSequenceNumber()); + } else { verify(this.mockSeqNrStorage, never()).queueUpForSave(any(SequenceNumberMap.class), anyLong()); } } From a34488b735180fca4999a39f3c0c99f52def840d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 15:12:52 -0800 Subject: [PATCH 20/51] [TESTS] Add unit tests for processGetDataResponse Add a full set of unit tests that uncovered some unexpected behavior w.r.t. signalers. --- .../network/p2p/storage/P2PDataStorage.java | 4 +- .../P2PDataStorageProcessGetDataResponse.java | 276 ++++++++++++++++++ 2 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 6a70f68ad26..fca70ae96ad 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -118,7 +118,9 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers @VisibleForTesting public static int CHECK_TTL_INTERVAL_SEC = 60; - private static boolean initialRequestApplied = false; + // TODO: Remove static + @VisibleForTesting + static boolean initialRequestApplied = false; private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java new file mode 100644 index 00000000000..37b76542fd5 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -0,0 +1,276 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.mockito.MockitoAnnotations; + +public class P2PDataStorageProcessGetDataResponse { + private TestState testState; + + private NodeAddress peerNodeAddress; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + this.testState = new TestState(); + P2PDataStorage.initialRequestApplied = false; + + this.peerNodeAddress = new NodeAddress("peer", 8080); + } + + static private GetDataResponse buildGetDataResponse(PersistableNetworkPayload persistableNetworkPayload) { + return buildGetDataResponse(Collections.emptyList(), Collections.singletonList(persistableNetworkPayload)); + } + + static private GetDataResponse buildGetDataResponse(ProtectedStorageEntry protectedStorageEntry) { + return buildGetDataResponse(Collections.singletonList(protectedStorageEntry), Collections.emptyList()); + } + + static private GetDataResponse buildGetDataResponse( + List protectedStorageEntries, + List persistableNetworkPayloads) { + return new GetDataResponse( + new HashSet<>(protectedStorageEntries), + persistableNetworkPayloads == null ? null : new HashSet<>(persistableNetworkPayloads), + 1, + false); + } + + /** + * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state + * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers + * is not supported. + */ + private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(1); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayloadStub + implements LazyProcessedPayload { + + LazyPersistableNetworkPayloadStub(byte[] hash) { + super(hash); + } + + LazyPersistableNetworkPayloadStub(boolean validHashSize) { + super(validHashSize); + } + } + + // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal + // XXXBUGXXX: We signal listeners w/ non LazyProcessedPayloads + @Test + public void processGetDataResponse_newPNPUpdatesState() { + PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, true, true, false, false); + } + + // TESTCASE: GetDataResponse w/ invalid PNP does nothing (LazyProcessed) + @Test + public void processGetDataResponse_newInvalidPNPDoesNothing() { + PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(false); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ existing PNP changes no state + @Test + public void processGetDataResponse_duplicatePNPDoesNothing() { + PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); + this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, + this.peerNodeAddress, false, false, false, false); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal (LazyProcessedPayload) + @Test + public void processGetDataResponse_newPNPUpdatesState_LazyProcessed() { + PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, true, false, false, false); + } + + // TESTCASE: GetDataResponse w/ existing PNP changes no state (LazyProcessedPayload) + @Test + public void processGetDataResponse_duplicatePNPDoesNothing_LazyProcessed() { + PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); + this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, + this.peerNodeAddress, false, false, false, false); + + GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ null PNP changes no state + @Test + public void processGetDataResponse_nullPNPSetDoesNothing() { + PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); + GetDataResponse getDataResponse = buildGetDataResponse(Collections.emptyList(), null); + + TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, persistableNetworkPayload, false, false, false, false); + } + + // TESTCASE: Second call to processGetDataResponse adds PNP for non-LazyProcessedPayloads + @Test + public void processGetDataResponse_secondProcessNewPNPUpdatesState() { + PersistableNetworkPayload addFromFirstProcess = new PersistableNetworkPayloadStub(new byte[] { 1 }); + GetDataResponse getDataResponse = buildGetDataResponse(addFromFirstProcess); + + TestState.SavedTestState beforeState = this.testState.saveTestState(addFromFirstProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromFirstProcess, true, true, false, false); + + PersistableNetworkPayload addFromSecondProcess = new PersistableNetworkPayloadStub(new byte[] { 2 }); + getDataResponse = buildGetDataResponse(addFromSecondProcess); + beforeState = this.testState.saveTestState(addFromSecondProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromSecondProcess, true, true, false, false); + } + + // TESTCASE: Second call to processGetDataResponse does not add any PNP (LazyProcessed) + @Test + public void processGetDataResponse_secondProcessNoPNPUpdates_LazyProcessed() { + PersistableNetworkPayload addFromFirstProcess = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); + GetDataResponse getDataResponse = buildGetDataResponse(addFromFirstProcess); + + TestState.SavedTestState beforeState = this.testState.saveTestState(addFromFirstProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromFirstProcess, true, false, false, false); + + PersistableNetworkPayload addFromSecondProcess = new LazyPersistableNetworkPayloadStub(new byte[] { 2 }); + getDataResponse = buildGetDataResponse(addFromSecondProcess); + beforeState = this.testState.saveTestState(addFromSecondProcess); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyPersistableAdd( + beforeState, addFromSecondProcess, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ missing PSE is added with no broadcast or listener signal + // XXXBUGXXX: We signal listeners for all ProtectedStorageEntrys + @Test + public void processGetDataResponse_newPSEUpdatesState() throws NoSuchAlgorithmException { + ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); + GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); + + TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, false, true, false); + } + + // TESTCASE: GetDataResponse w/ existing PSE changes no state + @Test + public void processGetDataResponse_duplicatePSEDoesNothing() throws NoSuchAlgorithmException { + ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); + this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, this.peerNodeAddress, null, false); + + GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); + + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, false, false, false, false, false); + } + + // TESTCASE: GetDataResponse w/ missing PSE is added with no broadcast or listener signal + // XXXBUGXXX: We signal listeners for all ProtectedStorageEntrys + @Test + public void processGetDataResponse_secondCallNewPSEUpdatesState() throws NoSuchAlgorithmException { + ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); + GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); + + TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, false, true, false); + + protectedStorageEntry = getProtectedStorageEntryForAdd(); + getDataResponse = buildGetDataResponse(protectedStorageEntry); + beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); + this.testState.verifyProtectedStorageAdd( + beforeState, protectedStorageEntry, true, true, false, true, false); + } +} From 3d6e9fbef587ceaf0691808bc96db158f9029248 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 15:14:19 -0800 Subject: [PATCH 21/51] Remove static from initialRequestApplied Previously, multiple handlers needed to signal off one global variable. Now, that this check is inside the singleton P2PDataStorage, make it non-static and private. --- .../main/java/bisq/network/p2p/storage/P2PDataStorage.java | 4 +--- .../p2p/storage/P2PDataStorageProcessGetDataResponse.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index fca70ae96ad..786a236cb8e 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -118,9 +118,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers @VisibleForTesting public static int CHECK_TTL_INTERVAL_SEC = 60; - // TODO: Remove static - @VisibleForTesting - static boolean initialRequestApplied = false; + private boolean initialRequestApplied = false; private final Broadcaster broadcaster; private final AppendOnlyDataStoreService appendOnlyDataStoreService; diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java index 37b76542fd5..c6b349183b5 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -52,7 +52,6 @@ public class P2PDataStorageProcessGetDataResponse { public void setUp() { MockitoAnnotations.initMocks(this); this.testState = new TestState(); - P2PDataStorage.initialRequestApplied = false; this.peerNodeAddress = new NodeAddress("peer", 8080); } From f92893b097add8fb67c357893e3294d724f0c441 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 15:43:12 -0800 Subject: [PATCH 22/51] [TESTS] Write synchronization integration tests Write a few integration test that exercises the exercise interesting synchronization states including the lost remove bug. This fails with the proper validation, but will pass at the end of the new feature development. --- .../P2PDataStorageGetDataIntegrationTest.java | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java new file mode 100644 index 00000000000..3b40575c633 --- /dev/null +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java @@ -0,0 +1,193 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage; + +import bisq.network.p2p.TestUtils; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.storage.mocks.PersistableExpirableProtectedStoragePayloadStub; +import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.app.Capabilities; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Assert; +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class P2PDataStorageGetDataIntegrationTest { + + /** + * Generates a unique ProtectedStorageEntry that is valid for add and remove. + */ + private ProtectedStorageEntry getProtectedStorageEntry() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + return getProtectedStorageEntry( + ownerKeys.getPublic(), new ProtectedStoragePayloadStub(ownerKeys.getPublic()), 1); + } + + private ProtectedStorageEntry getProtectedStorageEntry( + PublicKey ownerPubKey, + ProtectedStoragePayload protectedStoragePayload, + int sequenceNumber) { + ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); + when(stub.getOwnerPubKey()).thenReturn(ownerPubKey); + when(stub.isValidForAddOperation()).thenReturn(true); + when(stub.isValidForRemoveOperation()).thenReturn(true); + when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); + when(stub.getSequenceNumber()).thenReturn(sequenceNumber); + when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); + + return stub; + } + + // TESTCASE: Basic synchronization of a ProtectedStorageEntry works between a seed node and client node + @Test + public void basicSynchronizationWorks() throws NoSuchAlgorithmException { + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + ProtectedStorageEntry onSeedNode = getProtectedStorageEntry(); + seedNode.addProtectedStorageEntry(onSeedNode, null, null, false); + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(onSeedNode); + clientNode.processGetDataResponse(getDataResponse, null); + + clientNodeTestState.verifyProtectedStorageAdd( + beforeState, onSeedNode, true, true, false, true, false); + } + + // TESTCASE: Synchronization after peer restart works for in-memory ProtectedStorageEntrys + @Test + public void basicSynchronizationWorksAfterRestartTransient() throws NoSuchAlgorithmException { + ProtectedStorageEntry transientEntry = getProtectedStorageEntry(); + + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + seedNode.addProtectedStorageEntry(transientEntry, null, null, false); + + clientNode.addProtectedStorageEntry(transientEntry, null, null, false); + + clientNodeTestState.simulateRestart(); + clientNode = clientNodeTestState.mockedStorage; + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(transientEntry); + clientNode.processGetDataResponse(getDataResponse, null); + + clientNodeTestState.verifyProtectedStorageAdd( + beforeState, transientEntry, true, true, false, true, false); + } + + // TESTCASE: Synchronization after peer restart works for in-memory ProtectedStorageEntrys + @Test + public void basicSynchronizationWorksAfterRestartPersistent() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + ProtectedStoragePayload persistentPayload = + new PersistableExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry persistentEntry = getProtectedStorageEntry(ownerKeys.getPublic(), persistentPayload, 1); + + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + seedNode.addProtectedStorageEntry(persistentEntry, null, null, false); + + clientNode.addProtectedStorageEntry(persistentEntry, null, null, false); + + clientNodeTestState.simulateRestart(); + clientNode = clientNodeTestState.mockedStorage; + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(persistentEntry); + clientNode.processGetDataResponse(getDataResponse, null); + + clientNodeTestState.verifyProtectedStorageAdd( + beforeState, persistentEntry, false, false, false, false, false); + Assert.assertTrue(clientNodeTestState.mockedStorage.getMap().containsValue(persistentEntry)); + } + + // TESTCASE: Removes seen only by the seednode should be replayed on the client node + // during startup + // XXXBUGXXX: #3610 Lost removes are never replayed. + @Test + public void lostRemoveNeverUpdated() throws NoSuchAlgorithmException { + TestState seedNodeTestState = new TestState(); + P2PDataStorage seedNode = seedNodeTestState.mockedStorage; + + TestState clientNodeTestState = new TestState(); + P2PDataStorage clientNode = clientNodeTestState.mockedStorage; + + // Both nodes see the add + KeyPair ownerKeys = TestUtils.generateKeyPair(); + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry onSeedNodeAndClientNode = getProtectedStorageEntry( + ownerKeys.getPublic(), protectedStoragePayload, 1); + seedNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null, false); + clientNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null, false); + + // Seed node sees the remove, but client node does not + seedNode.remove(getProtectedStorageEntry( + ownerKeys.getPublic(), protectedStoragePayload, 2), null, false); + + GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); + + GetDataResponse getDataResponse = seedNode.buildGetDataResponse( + getDataRequest, 1, new AtomicBoolean(), new AtomicBoolean(), new Capabilities()); + + TestState.SavedTestState beforeState = clientNodeTestState.saveTestState(onSeedNodeAndClientNode); + clientNode.processGetDataResponse(getDataResponse, null); + + // Should succeed + clientNodeTestState.verifyProtectedStorageRemove( + beforeState, onSeedNodeAndClientNode, false, false, false, false, false); + } +} From 5db128587f7b8d6913a9eda27a83224294424e4e Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 09:13:36 -0800 Subject: [PATCH 23/51] [REFACTOR] Clean up processGetDataResponse - Add more comments - Use Clock instead of System - Remove unnecessary AtomicInteger --- .../network/p2p/storage/P2PDataStorage.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 786a236cb8e..fa24c4c45c6 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -328,21 +328,15 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); long ts2 = System.currentTimeMillis(); - AtomicInteger counter = new AtomicInteger(); dataSet.forEach(e -> { // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless addProtectedStorageEntry(e, sender, null, false, false); - counter.getAndIncrement(); }); - log.info("Processing {} protectedStorageEntries took {} ms.", counter.get(), System.currentTimeMillis() - ts2); + log.info("Processing {} protectedStorageEntries took {} ms.", dataSet.size(), this.clock.millis() - ts2); - /* // engage the firstRequest logic only if we are a seed node. Normal clients get here twice at most. - if (!Capabilities.app.containsAll(Capability.SEED_NODE)) - firstRequest = true;*/ - - if (persistableNetworkPayloadSet != null /*&& firstRequest*/) { - ts2 = System.currentTimeMillis(); + if (persistableNetworkPayloadSet != null) { + ts2 = this.clock.millis(); persistableNetworkPayloadSet.forEach(e -> { if (e instanceof LazyProcessedPayload) { // We use an optimized method as many checks are not required in that case to avoid @@ -362,13 +356,14 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress false, false, false); } }); - - // We set initialRequestApplied to true after the loop, otherwise we would only process 1 entry - initialRequestApplied = true; - log.info("Processing {} persistableNetworkPayloads took {} ms.", - persistableNetworkPayloadSet.size(), System.currentTimeMillis() - ts2); + persistableNetworkPayloadSet.size(), this.clock.millis() - ts2); } + + // We only process PersistableNetworkPayloads implementing LazyProcessedPayload once. It can cause performance + // issues and since the data is rarely out of sync it is not worth it to apply them from multiple peers during + // startup. + initialRequestApplied = true; } /////////////////////////////////////////////////////////////////////////////////////////// From ecae31eddb8c4cf0a88d89da52517453ebe9e088 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 09:22:12 -0800 Subject: [PATCH 24/51] [RENAME] LazyProcessedPayload to ProcessOncePersistableNetworkPayload Name is left over from previous implementation. Change it to be more relevant to the current code and update comments to indicate the current usage. --- .../java/bisq/core/account/sign/SignedWitness.java | 4 ++-- .../bisq/core/account/witness/AccountAgeWitness.java | 4 ++-- .../proposal/storage/temp/TempProposalPayload.java | 4 ++-- .../bisq/core/trade/statistics/TradeStatistics.java | 4 ++-- .../bisq/core/trade/statistics/TradeStatistics2.java | 4 ++-- .../bisq/network/p2p/storage/P2PDataStorage.java | 6 +++--- ...ava => ProcessOncePersistableNetworkPayload.java} | 7 ++++--- .../peers/getdata/GetDataRequestOnMessageTest.java | 4 ++-- .../P2PDataStorageProcessGetDataResponse.java | 12 ++++++------ 9 files changed, 25 insertions(+), 24 deletions(-) rename p2p/src/main/java/bisq/network/p2p/storage/payload/{LazyProcessedPayload.java => ProcessOncePersistableNetworkPayload.java} (72%) diff --git a/core/src/main/java/bisq/core/account/sign/SignedWitness.java b/core/src/main/java/bisq/core/account/sign/SignedWitness.java index e5af940470c..d2268400ac6 100644 --- a/core/src/main/java/bisq/core/account/sign/SignedWitness.java +++ b/core/src/main/java/bisq/core/account/sign/SignedWitness.java @@ -20,7 +20,7 @@ import bisq.network.p2p.storage.P2PDataStorage; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.common.app.Capabilities; @@ -45,7 +45,7 @@ // Supports signatures made from EC key (arbitrators) and signature created with DSA key. @Slf4j @Value -public class SignedWitness implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope, +public class SignedWitness implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, PersistableEnvelope, DateTolerantPayload, CapabilityRequiringPayload { public enum VerificationMethod { diff --git a/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java b/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java index 9578df22973..07f05096602 100644 --- a/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java +++ b/core/src/main/java/bisq/core/account/witness/AccountAgeWitness.java @@ -19,7 +19,7 @@ import bisq.network.p2p.storage.P2PDataStorage; import bisq.network.p2p.storage.payload.DateTolerantPayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.common.proto.persistable.PersistableEnvelope; @@ -40,7 +40,7 @@ // so only the newly added objects since the last release will be retrieved over the P2P network. @Slf4j @Value -public class AccountAgeWitness implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope, DateTolerantPayload { +public class AccountAgeWitness implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, PersistableEnvelope, DateTolerantPayload { private static final long TOLERANCE = TimeUnit.DAYS.toMillis(1); private final byte[] hash; // Ripemd160(Sha256(concatenated accountHash, signature and sigPubKey)); 20 bytes diff --git a/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java b/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java index 6ea17a819d1..f5334fa6030 100644 --- a/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java +++ b/core/src/main/java/bisq/core/dao/governance/proposal/storage/temp/TempProposalPayload.java @@ -20,7 +20,7 @@ import bisq.core.dao.state.model.governance.Proposal; import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.crypto.Sig; @@ -55,7 +55,7 @@ @Getter @EqualsAndHashCode @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class TempProposalPayload implements LazyProcessedPayload, ProtectedStoragePayload, +public class TempProposalPayload implements ProcessOncePersistableNetworkPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { protected final Proposal proposal; diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java index 54412a5de10..41c77aeb36e 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java @@ -24,7 +24,7 @@ import bisq.core.offer.OfferPayload; import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.crypto.Sig; @@ -60,7 +60,7 @@ @Slf4j @EqualsAndHashCode(exclude = {"signaturePubKeyBytes"}) @Value -public final class TradeStatistics implements LazyProcessedPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { +public final class TradeStatistics implements ProcessOncePersistableNetworkPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { private final OfferPayload.Direction direction; private final String baseCurrency; private final String counterCurrency; diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index 601284f4e74..ba1f6abaf25 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -25,7 +25,7 @@ import bisq.core.offer.OfferUtil; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.common.app.Capabilities; @@ -63,7 +63,7 @@ @Slf4j @Value -public final class TradeStatistics2 implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope, CapabilityRequiringPayload { +public final class TradeStatistics2 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, PersistableEnvelope, CapabilityRequiringPayload { //We don't support arbitrators anymore so this entry will be only for pre v1.2. trades @Deprecated diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index fa24c4c45c6..d143ba1f4c7 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -39,7 +39,7 @@ import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.MailboxStoragePayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry; @@ -338,7 +338,7 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress if (persistableNetworkPayloadSet != null) { ts2 = this.clock.millis(); persistableNetworkPayloadSet.forEach(e -> { - if (e instanceof LazyProcessedPayload) { + if (e instanceof ProcessOncePersistableNetworkPayload) { // We use an optimized method as many checks are not required in that case to avoid // performance issues. // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). @@ -360,7 +360,7 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress persistableNetworkPayloadSet.size(), this.clock.millis() - ts2); } - // We only process PersistableNetworkPayloads implementing LazyProcessedPayload once. It can cause performance + // We only process PersistableNetworkPayloads implementing ProcessOncePersistableNetworkPayload once. It can cause performance // issues and since the data is rarely out of sync it is not worth it to apply them from multiple peers during // startup. initialRequestApplied = true; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/LazyProcessedPayload.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProcessOncePersistableNetworkPayload.java similarity index 72% rename from p2p/src/main/java/bisq/network/p2p/storage/payload/LazyProcessedPayload.java rename to p2p/src/main/java/bisq/network/p2p/storage/payload/ProcessOncePersistableNetworkPayload.java index f63d1cf9b62..70fe1c35abe 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/LazyProcessedPayload.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProcessOncePersistableNetworkPayload.java @@ -20,8 +20,9 @@ import bisq.common.Payload; /** - * Marker interface for payload which gets delayed processed at startup so we don't hit performance too much. - * Used for TradeStatistics and AccountAgeWitness. + * Marker interface for PersistableNetworkPayloads that are only added during the FIRST call to + * P2PDataStorage::processDataResponse. This improves performance for objects that don't go out + * of sync frequently. */ -public interface LazyProcessedPayload extends Payload { +public interface ProcessOncePersistableNetworkPayload extends Payload { } diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java index 2ac1c25b884..deaf85a85b1 100644 --- a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java +++ b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java @@ -27,7 +27,7 @@ import bisq.network.p2p.storage.TestState; import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; @@ -99,7 +99,7 @@ private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgo } static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayloadStub - implements LazyProcessedPayload { + implements ProcessOncePersistableNetworkPayload { LazyPersistableNetworkPayloadStub(byte[] hash) { super(hash); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java index c6b349183b5..284ff197e81 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -22,7 +22,7 @@ import bisq.network.p2p.peers.getdata.messages.GetDataResponse; import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; -import bisq.network.p2p.storage.payload.LazyProcessedPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; import bisq.network.p2p.storage.payload.ProtectedStoragePayload; @@ -95,7 +95,7 @@ private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgo } static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayloadStub - implements LazyProcessedPayload { + implements ProcessOncePersistableNetworkPayload { LazyPersistableNetworkPayloadStub(byte[] hash) { super(hash); @@ -107,7 +107,7 @@ static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayload } // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal - // XXXBUGXXX: We signal listeners w/ non LazyProcessedPayloads + // XXXBUGXXX: We signal listeners w/ non ProcessOncePersistableNetworkPayloads @Test public void processGetDataResponse_newPNPUpdatesState() { PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); @@ -148,7 +148,7 @@ public void processGetDataResponse_duplicatePNPDoesNothing() { beforeState, persistableNetworkPayload, false, false, false, false); } - // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal (LazyProcessedPayload) + // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal (ProcessOncePersistableNetworkPayload) @Test public void processGetDataResponse_newPNPUpdatesState_LazyProcessed() { PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); @@ -161,7 +161,7 @@ public void processGetDataResponse_newPNPUpdatesState_LazyProcessed() { beforeState, persistableNetworkPayload, true, false, false, false); } - // TESTCASE: GetDataResponse w/ existing PNP changes no state (LazyProcessedPayload) + // TESTCASE: GetDataResponse w/ existing PNP changes no state (ProcessOncePersistableNetworkPayload) @Test public void processGetDataResponse_duplicatePNPDoesNothing_LazyProcessed() { PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); @@ -188,7 +188,7 @@ public void processGetDataResponse_nullPNPSetDoesNothing() { beforeState, persistableNetworkPayload, false, false, false, false); } - // TESTCASE: Second call to processGetDataResponse adds PNP for non-LazyProcessedPayloads + // TESTCASE: Second call to processGetDataResponse adds PNP for non-ProcessOncePersistableNetworkPayloads @Test public void processGetDataResponse_secondProcessNewPNPUpdatesState() { PersistableNetworkPayload addFromFirstProcess = new PersistableNetworkPayloadStub(new byte[] { 1 }); From a0fae120687ed83ec3c7ed3628c27f018d4209cc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 09:50:48 -0800 Subject: [PATCH 25/51] Remove @Nullable around persistableNetworkPayloadSet Checking for null creates hard-to-read code and it is simpler to just create an empty set if we receive a pre-v0.6 GetDataResponse protobuf message that does not have the field set. --- .../bisq/monitor/metric/P2PMarketStats.java | 19 +++----- .../monitor/metric/P2PSeedNodeSnapshot.java | 19 +++----- .../p2p/peers/getdata/RequestDataHandler.java | 20 ++++----- .../getdata/messages/GetDataResponse.java | 24 +++++----- .../network/p2p/storage/P2PDataStorage.java | 44 +++++++++---------- .../P2PDataStorageProcessGetDataResponse.java | 14 +----- 6 files changed, 55 insertions(+), 85 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java index edebbf13d44..21a17fd9945 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java @@ -221,20 +221,15 @@ protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection conne versions.log(protectedStoragePayload); }); - Set persistableNetworkPayloadSet = dataResponse - .getPersistableNetworkPayloadSet(); - if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { + dataResponse.getPersistableNetworkPayloadSet().forEach(persistableNetworkPayload -> { + // memorize message hashes + //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; + //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); - // memorize message hashes - //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; - //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); + //hashes.add(bytes); - //hashes.add(bytes); - - hashes.add(persistableNetworkPayload.getHash()); - }); - } + hashes.add(persistableNetworkPayload.getHash()); + }); bucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), result); versionBucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), versions); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java index 77462aa4e5e..aefb227189c 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java @@ -333,20 +333,15 @@ protected boolean treatMessage(NetworkEnvelope networkEnvelope, Connection conne result.log(protectedStoragePayload); }); - Set persistableNetworkPayloadSet = dataResponse - .getPersistableNetworkPayloadSet(); - if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { + dataResponse.getPersistableNetworkPayloadSet().forEach(persistableNetworkPayload -> { + // memorize message hashes + //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; + //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); - // memorize message hashes - //Byte[] bytes = new Byte[persistableNetworkPayload.getHash().length]; - //Arrays.setAll(bytes, n -> persistableNetworkPayload.getHash()[n]); + //hashes.add(bytes); - //hashes.add(bytes); - - hashes.add(persistableNetworkPayload.getHash()); - }); - } + hashes.add(persistableNetworkPayload.getHash()); + }); bucketsPerHost.put(connection.getPeersNodeAddressProperty().getValue(), result); return true; diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index 01c0f00361a..1685a96a659 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -248,24 +248,20 @@ private void logContents(NetworkEnvelope networkEnvelope, payloadByClassName.get(className).add(protectedStoragePayload); }); + persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { + // For logging different data types + String className = persistableNetworkPayload.getClass().getSimpleName(); + if (!payloadByClassName.containsKey(className)) + payloadByClassName.put(className, new HashSet<>()); - if (persistableNetworkPayloadSet != null) { - persistableNetworkPayloadSet.forEach(persistableNetworkPayload -> { - // For logging different data types - String className = persistableNetworkPayload.getClass().getSimpleName(); - if (!payloadByClassName.containsKey(className)) - payloadByClassName.put(className, new HashSet<>()); - - payloadByClassName.get(className).add(persistableNetworkPayload); - }); - } + payloadByClassName.get(className).add(persistableNetworkPayload); + }); // Log different data types StringBuilder sb = new StringBuilder(); sb.append("\n#################################################################\n"); sb.append("Connected to node: " + peersNodeAddress.getFullAddress() + "\n"); - final int items = dataSet.size() + - (persistableNetworkPayloadSet != null ? persistableNetworkPayloadSet.size() : 0); + final int items = dataSet.size() + persistableNetworkPayloadSet.size(); sb.append("Received ").append(items).append(" instances\n"); payloadByClassName.forEach((key, value) -> sb.append(key) .append(": ") diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java index 0386d1abe1b..bb6e9b83aeb 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java @@ -37,6 +37,8 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + import javax.annotation.Nullable; @Slf4j @@ -47,8 +49,7 @@ public final class GetDataResponse extends NetworkEnvelope implements SupportedC private final Set dataSet; // Set of PersistableNetworkPayload objects - // We added that in v 0.6 and we would get a null object from older peers, so keep it annotated with @Nullable - @Nullable + // We added that in v 0.6 and the fromProto code will create an empty HashSet if it doesn't exist private final Set persistableNetworkPayloadSet; private final int requestNonce; @@ -57,7 +58,7 @@ public final class GetDataResponse extends NetworkEnvelope implements SupportedC private final Capabilities supportedCapabilities; public GetDataResponse(Set dataSet, - @Nullable Set persistableNetworkPayloadSet, + @NotNull Set persistableNetworkPayloadSet, int requestNonce, boolean isGetUpdatedDataResponse) { this(dataSet, @@ -73,7 +74,7 @@ public GetDataResponse(Set dataSet, /////////////////////////////////////////////////////////////////////////////////////////// private GetDataResponse(Set dataSet, - @Nullable Set persistableNetworkPayloadSet, + @NotNull Set persistableNetworkPayloadSet, int requestNonce, boolean isGetUpdatedDataResponse, @Nullable Capabilities supportedCapabilities, @@ -101,12 +102,12 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .build()) .collect(Collectors.toList())) .setRequestNonce(requestNonce) - .setIsGetUpdatedDataResponse(isGetUpdatedDataResponse); + .setIsGetUpdatedDataResponse(isGetUpdatedDataResponse) + .addAllPersistableNetworkPayloadItems(persistableNetworkPayloadSet.stream() + .map(PersistableNetworkPayload::toProtoMessage) + .collect(Collectors.toList())); Optional.ofNullable(supportedCapabilities).ifPresent(e -> builder.addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities))); - Optional.ofNullable(persistableNetworkPayloadSet).ifPresent(set -> builder.addAllPersistableNetworkPayloadItems(set.stream() - .map(PersistableNetworkPayload::toProtoMessage) - .collect(Collectors.toList()))); protobuf.NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetDataResponse(builder) @@ -124,14 +125,11 @@ public static GetDataResponse fromProto(protobuf.GetDataResponse proto, .map(entry -> (ProtectedStorageEntry) resolver.fromProto(entry)) .collect(Collectors.toSet())); - Set persistableNetworkPayloadSet = proto.getPersistableNetworkPayloadItemsList().isEmpty() ? - null : - new HashSet<>( - proto.getPersistableNetworkPayloadItemsList().stream() + Set persistableNetworkPayloadSet = new HashSet<>( + proto.getPersistableNetworkPayloadItemsList().stream() .map(e -> (PersistableNetworkPayload) resolver.fromProto(e)) .collect(Collectors.toSet())); - //PersistableNetworkPayload return new GetDataResponse(dataSet, persistableNetworkPayloadSet, proto.getRequestNonce(), diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index d143ba1f4c7..551b15bae49 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -335,30 +335,28 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress }); log.info("Processing {} protectedStorageEntries took {} ms.", dataSet.size(), this.clock.millis() - ts2); - if (persistableNetworkPayloadSet != null) { - ts2 = this.clock.millis(); - persistableNetworkPayloadSet.forEach(e -> { - if (e instanceof ProcessOncePersistableNetworkPayload) { - // We use an optimized method as many checks are not required in that case to avoid - // performance issues. - // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). - // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all - // trade stats stats and all account age witness data. - - // We only apply it once from first response - if (!initialRequestApplied) { - addPersistableNetworkPayloadFromInitialRequest(e); - - } - } else { - // We don't broadcast here as we are only connected to the seed node and would be pointless - addPersistableNetworkPayload(e, sender, false, - false, false, false); + ts2 = this.clock.millis(); + persistableNetworkPayloadSet.forEach(e -> { + if (e instanceof ProcessOncePersistableNetworkPayload) { + // We use an optimized method as many checks are not required in that case to avoid + // performance issues. + // Processing 82645 items took now 61 ms compared to earlier version where it took ages (> 2min). + // Usually we only get about a few hundred or max. a few 1000 items. 82645 is all + // trade stats stats and all account age witness data. + + // We only apply it once from first response + if (!initialRequestApplied) { + addPersistableNetworkPayloadFromInitialRequest(e); + } - }); - log.info("Processing {} persistableNetworkPayloads took {} ms.", - persistableNetworkPayloadSet.size(), this.clock.millis() - ts2); - } + } else { + // We don't broadcast here as we are only connected to the seed node and would be pointless + addPersistableNetworkPayload(e, sender, false, + false, false, false); + } + }); + log.info("Processing {} persistableNetworkPayloads took {} ms.", + persistableNetworkPayloadSet.size(), this.clock.millis() - ts2); // We only process PersistableNetworkPayloads implementing ProcessOncePersistableNetworkPayload once. It can cause performance // issues and since the data is rarely out of sync it is not worth it to apply them from multiple peers during diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java index 284ff197e81..c7177b005d7 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -69,7 +69,7 @@ static private GetDataResponse buildGetDataResponse( List persistableNetworkPayloads) { return new GetDataResponse( new HashSet<>(protectedStorageEntries), - persistableNetworkPayloads == null ? null : new HashSet<>(persistableNetworkPayloads), + new HashSet<>(persistableNetworkPayloads), 1, false); } @@ -176,18 +176,6 @@ public void processGetDataResponse_duplicatePNPDoesNothing_LazyProcessed() { beforeState, persistableNetworkPayload, false, false, false, false); } - // TESTCASE: GetDataResponse w/ null PNP changes no state - @Test - public void processGetDataResponse_nullPNPSetDoesNothing() { - PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); - GetDataResponse getDataResponse = buildGetDataResponse(Collections.emptyList(), null); - - TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); - this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); - this.testState.verifyPersistableAdd( - beforeState, persistableNetworkPayload, false, false, false, false); - } - // TESTCASE: Second call to processGetDataResponse adds PNP for non-ProcessOncePersistableNetworkPayloads @Test public void processGetDataResponse_secondProcessNewPNPUpdatesState() { From c503bcbaed54175fc35400175befdc26145896f7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 11:36:53 -0800 Subject: [PATCH 26/51] Remove @Nullable around supportedCapabilities in GetDataResponse The only two users of this constructor are the fromProto path which already creates an empty Capabilities object if one is not provided and the internal usage of Capabilities.app which is initialized to empty. Remove the @Nullable so future readers aren't confused. --- .../getdata/messages/GetDataResponse.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java index bb6e9b83aeb..a86d6f27834 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java @@ -29,7 +29,6 @@ import bisq.common.proto.network.NetworkProtoResolver; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -39,8 +38,6 @@ import org.jetbrains.annotations.NotNull; -import javax.annotation.Nullable; - @Slf4j @EqualsAndHashCode(callSuper = true) @Value @@ -54,10 +51,9 @@ public final class GetDataResponse extends NetworkEnvelope implements SupportedC private final int requestNonce; private final boolean isGetUpdatedDataResponse; - @Nullable private final Capabilities supportedCapabilities; - public GetDataResponse(Set dataSet, + public GetDataResponse(@NotNull Set dataSet, @NotNull Set persistableNetworkPayloadSet, int requestNonce, boolean isGetUpdatedDataResponse) { @@ -73,11 +69,11 @@ public GetDataResponse(Set dataSet, // PROTO BUFFER /////////////////////////////////////////////////////////////////////////////////////////// - private GetDataResponse(Set dataSet, + private GetDataResponse(@NotNull Set dataSet, @NotNull Set persistableNetworkPayloadSet, int requestNonce, boolean isGetUpdatedDataResponse, - @Nullable Capabilities supportedCapabilities, + @NotNull Capabilities supportedCapabilities, int messageVersion) { super(messageVersion); @@ -101,13 +97,12 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .setProtectedStorageEntry((protobuf.ProtectedStorageEntry) protectedStorageEntry.toProtoMessage()) .build()) .collect(Collectors.toList())) - .setRequestNonce(requestNonce) - .setIsGetUpdatedDataResponse(isGetUpdatedDataResponse) .addAllPersistableNetworkPayloadItems(persistableNetworkPayloadSet.stream() .map(PersistableNetworkPayload::toProtoMessage) - .collect(Collectors.toList())); - - Optional.ofNullable(supportedCapabilities).ifPresent(e -> builder.addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities))); + .collect(Collectors.toList())) + .setRequestNonce(requestNonce) + .setIsGetUpdatedDataResponse(isGetUpdatedDataResponse) + .addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities)); protobuf.NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setGetDataResponse(builder) From b1a06fe4e20586448e8e8a7a6be45653ee0cd10b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 10:16:47 -0800 Subject: [PATCH 27/51] Remove @Nullable around supportedCapabilities in PreliminaryGetDataRequest The only two users of this constructor are the fromProto path which now creates an empty Capabilities object similar to GetDataResponse. The other internal usage of Capabilities.app which is initialized to empty. --- .../messages/PreliminaryGetDataRequest.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java index 0702b0f1756..db0e51eb7f6 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java @@ -26,7 +26,6 @@ import com.google.protobuf.ByteString; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -34,9 +33,7 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nullable; - - +import org.jetbrains.annotations.NotNull; import protobuf.NetworkEnvelope; @@ -44,11 +41,10 @@ @EqualsAndHashCode(callSuper = true) @Value public final class PreliminaryGetDataRequest extends GetDataRequest implements AnonymousMessage, SupportedCapabilitiesMessage { - @Nullable private final Capabilities supportedCapabilities; public PreliminaryGetDataRequest(int nonce, - Set excludedKeys) { + @NotNull Set excludedKeys) { this(nonce, excludedKeys, Capabilities.app, Version.getP2PMessageVersion()); } @@ -58,8 +54,8 @@ public PreliminaryGetDataRequest(int nonce, /////////////////////////////////////////////////////////////////////////////////////////// private PreliminaryGetDataRequest(int nonce, - Set excludedKeys, - @Nullable Capabilities supportedCapabilities, + @NotNull Set excludedKeys, + @NotNull Capabilities supportedCapabilities, int messageVersion) { super(messageVersion, nonce, excludedKeys); @@ -69,13 +65,12 @@ private PreliminaryGetDataRequest(int nonce, @Override public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { final protobuf.PreliminaryGetDataRequest.Builder builder = protobuf.PreliminaryGetDataRequest.newBuilder() + .addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities)) .setNonce(nonce) .addAllExcludedKeys(excludedKeys.stream() .map(ByteString::copyFrom) .collect(Collectors.toList())); - Optional.ofNullable(supportedCapabilities).ifPresent(e -> builder.addAllSupportedCapabilities(Capabilities.toIntList(supportedCapabilities))); - NetworkEnvelope proto = getNetworkEnvelopeBuilder() .setPreliminaryGetDataRequest(builder) .build(); @@ -85,13 +80,9 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { public static PreliminaryGetDataRequest fromProto(protobuf.PreliminaryGetDataRequest proto, int messageVersion) { log.info("Received a PreliminaryGetDataRequest with {} kB", proto.getSerializedSize() / 1000d); - Capabilities supportedCapabilities = proto.getSupportedCapabilitiesList().isEmpty() ? - null : - Capabilities.fromIntList(proto.getSupportedCapabilitiesList()); - return new PreliminaryGetDataRequest(proto.getNonce(), ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedKeysList()), - supportedCapabilities, + Capabilities.fromIntList(proto.getSupportedCapabilitiesList()), messageVersion); } } From 4fe19aeec20e3587571640f74b2541c49e6755c9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 10:33:12 -0800 Subject: [PATCH 28/51] [DEADCODE] Remove old request handler tests Now that all the implementations are unit tested in P2PDataStorage, the old tests can be removed. --- .../p2p/peers/getdata/RequestDataHandler.java | 4 +- .../getdata/GetDataRequestHandlerTest.java | 199 -------------- .../getdata/GetDataRequestOnMessageTest.java | 160 ----------- .../RequestDataHandlerRequestDataTest.java | 249 ------------------ .../bisq/network/p2p/storage/TestState.java | 5 +- 5 files changed, 2 insertions(+), 615 deletions(-) delete mode 100644 p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java delete mode 100644 p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java delete mode 100644 p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java index 1685a96a659..4d1ada32941 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java @@ -35,7 +35,6 @@ import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.network.NetworkPayload; -import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.SettableFuture; @@ -87,8 +86,7 @@ public interface Listener { private final PeerManager peerManager; private final Listener listener; private Timer timeoutTimer; - @VisibleForTesting - final int nonce = new Random().nextInt(); + private final int nonce = new Random().nextInt(); private boolean stopped; diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java deleted file mode 100644 index 10a7555e785..00000000000 --- a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestHandlerTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.network.p2p.peers.getdata; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.TestUtils; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; -import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; -import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.TestState; -import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; -import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; -import bisq.network.p2p.storage.payload.PersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.Proto; -import bisq.common.app.Capabilities; -import bisq.common.app.Capability; - -import com.google.common.util.concurrent.SettableFuture; - -import java.security.KeyPair; -import java.security.NoSuchAlgorithmException; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import static org.mockito.Mockito.*; - -public class GetDataRequestHandlerTest { - private TestState testState; - - @Mock - NetworkNode networkNode; - - private NodeAddress localNodeAddress; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - this.testState = new TestState(); - - this.localNodeAddress = new NodeAddress("localhost", 8080); - when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); - - // Set up basic capabilities to ensure message contains it. Ensure it is unique from other tests - // so we catch mismatch bugs. - Capabilities.app.addAll(Capability.DAO_FULL_NODE); - } - - /** - * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state - * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers - * is not supported. - */ - private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { - KeyPair ownerKeys = TestUtils.generateKeyPair(); - - ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); - - ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); - when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); - when(stub.isValidForAddOperation()).thenReturn(true); - when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); - when(stub.getSequenceNumber()).thenReturn(1); - when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); - - return stub; - } - - // TESTCASE: Construct a request that requires excluding duplicates and adding missing entrys for - // PersistableNetworkPayloads and ProtectedStorageEntrys to verify the correct keys are added to the response. - @Test - public void handle_PreliminaryGetDataRequestTest() throws NoSuchAlgorithmException { - - // Construct a seed node w/ 2 PersistableNetworkPayloads and 2 ProtectedStorageEntrys and a PreliminaryGetDataRequest - // that only contains 1 known PersistableNetworkPayload and 1 known ProtectedStorageEntry as well as 2 unknowns - PersistableNetworkPayload pnp_onSeedNodeNotInRequest = new PersistableNetworkPayloadStub(new byte[] { 1 }); - PersistableNetworkPayload pnp_onSeedNodeAndInRequest = new PersistableNetworkPayloadStub(new byte[] { 2 }); - PersistableNetworkPayload pnp_onlyInRequest = new PersistableNetworkPayloadStub(new byte[] { 3 }); - ProtectedStorageEntry pse_onSeedNodeNotInRequest = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry pse_onSeedNodeAndInRequest = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry pse_onlyInRequest = getProtectedStorageEntryForAdd(); - - this.testState.getMockedStorage().addPersistableNetworkPayload( - pnp_onSeedNodeNotInRequest, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addPersistableNetworkPayload( - pnp_onSeedNodeAndInRequest, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeNotInRequest, this.localNodeAddress, null, false); - this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeAndInRequest, this.localNodeAddress, null, false); - - SettableFuture sendFuture = mock(SettableFuture.class); - final ArgumentCaptor captor = ArgumentCaptor.forClass(GetDataResponse.class); - when(this.networkNode.sendMessage(any(Connection.class), captor.capture())).thenReturn(sendFuture); - - PreliminaryGetDataRequest getDataRequest = mock(PreliminaryGetDataRequest.class); - HashSet knownKeysSet = new HashSet<>(Arrays.asList( - pnp_onSeedNodeAndInRequest.getHash(), - pnp_onlyInRequest.getHash(), - P2PDataStorage.get32ByteHash(pse_onSeedNodeAndInRequest.getProtectedStoragePayload()), - P2PDataStorage.get32ByteHash(pse_onlyInRequest.getProtectedStoragePayload()))); - when(getDataRequest.getNonce()).thenReturn(1); - when(getDataRequest.getExcludedKeys()).thenReturn(knownKeysSet); - - Connection connection = mock(Connection.class); - when(connection.noCapabilityRequiredOrCapabilityIsSupported(any(Proto.class))).thenReturn(true); - - GetDataRequestHandler handler = - new GetDataRequestHandler(this.networkNode, this.testState.getMockedStorage(), null); - handler.handle(getDataRequest, connection); - - // Verify the request node is sent back only the 2 missing payloads - GetDataResponse getDataResponse = captor.getValue(); - Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); - Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); - Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); - Assert.assertFalse(getDataResponse.isGetUpdatedDataResponse()); - Assert.assertEquals(getDataResponse.getPersistableNetworkPayloadSet(), new HashSet<>(Collections.singletonList(pnp_onSeedNodeNotInRequest))); - Assert.assertEquals(getDataResponse.getDataSet(), new HashSet<>(Collections.singletonList(pse_onSeedNodeNotInRequest))); - } - - // TESTCASE: Same as above, but with an GetUpdatedDataRequest - @Test - public void handle_GetUpdatedDataRequestTest() throws NoSuchAlgorithmException { - - // Construct a seed node w/ 2 PersistableNetworkPayloads and 2 ProtectedStorageEntrys and a PreliminaryGetDataRequest - // that only contains 1 known PersistableNetworkPayload and 1 known ProtectedStorageEntry as well as 2 unknowns - PersistableNetworkPayload pnp_onSeedNodeNotInRequest = new PersistableNetworkPayloadStub(new byte[] { 1 }); - PersistableNetworkPayload pnp_onSeedNodeAndInRequest = new PersistableNetworkPayloadStub(new byte[] { 2 }); - PersistableNetworkPayload pnp_onlyInRequest = new PersistableNetworkPayloadStub(new byte[] { 3 }); - ProtectedStorageEntry pse_onSeedNodeNotInRequest = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry pse_onSeedNodeAndInRequest = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry pse_onlyInRequest = getProtectedStorageEntryForAdd(); - - this.testState.getMockedStorage().addPersistableNetworkPayload( - pnp_onSeedNodeNotInRequest, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addPersistableNetworkPayload( - pnp_onSeedNodeAndInRequest, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeNotInRequest, this.localNodeAddress, null, false); - this.testState.getMockedStorage().addProtectedStorageEntry(pse_onSeedNodeAndInRequest, this.localNodeAddress, null, false); - - SettableFuture sendFuture = mock(SettableFuture.class); - final ArgumentCaptor captor = ArgumentCaptor.forClass(GetDataResponse.class); - when(this.networkNode.sendMessage(any(Connection.class), captor.capture())).thenReturn(sendFuture); - - GetUpdatedDataRequest getDataRequest = mock(GetUpdatedDataRequest.class); - HashSet knownKeysSet = new HashSet<>(Arrays.asList( - pnp_onSeedNodeAndInRequest.getHash(), - pnp_onlyInRequest.getHash(), - P2PDataStorage.get32ByteHash(pse_onSeedNodeAndInRequest.getProtectedStoragePayload()), - P2PDataStorage.get32ByteHash(pse_onlyInRequest.getProtectedStoragePayload()))); - when(getDataRequest.getNonce()).thenReturn(1); - when(getDataRequest.getExcludedKeys()).thenReturn(knownKeysSet); - - Connection connection = mock(Connection.class); - when(connection.noCapabilityRequiredOrCapabilityIsSupported(any(Proto.class))).thenReturn(true); - - GetDataRequestHandler handler = - new GetDataRequestHandler(this.networkNode, this.testState.getMockedStorage(), null); - handler.handle(getDataRequest, connection); - - // Verify the request node is sent back only the 2 missing payloads - GetDataResponse getDataResponse = captor.getValue(); - Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); - Assert.assertEquals(getDataResponse.getSupportedCapabilities(), Capabilities.app); - Assert.assertEquals(getDataResponse.getRequestNonce(), getDataRequest.getNonce()); - Assert.assertTrue(getDataResponse.isGetUpdatedDataResponse()); - Assert.assertEquals(getDataResponse.getPersistableNetworkPayloadSet(), new HashSet<>(Collections.singletonList(pnp_onSeedNodeNotInRequest))); - Assert.assertEquals(getDataResponse.getDataSet(), new HashSet<>(Collections.singletonList(pse_onSeedNodeNotInRequest))); - } -} diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java deleted file mode 100644 index deaf85a85b1..00000000000 --- a/p2p/src/test/java/bisq/network/p2p/peers/getdata/GetDataRequestOnMessageTest.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.network.p2p.peers.getdata; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.TestUtils; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.peers.PeerManager; -import bisq.network.p2p.peers.getdata.messages.GetDataRequest; -import bisq.network.p2p.peers.getdata.messages.GetDataResponse; -import bisq.network.p2p.storage.TestState; -import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; -import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; -import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; -import bisq.network.p2p.storage.payload.PersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.app.Capabilities; -import bisq.common.app.Capability; - -import com.google.common.util.concurrent.SettableFuture; - -import java.security.KeyPair; -import java.security.NoSuchAlgorithmException; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Optional; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - - -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -public class GetDataRequestOnMessageTest { - private TestState testState; - - @Mock - NetworkNode networkNode; - - private NodeAddress localNodeAddress; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - this.testState = new TestState(); - - this.localNodeAddress = new NodeAddress("localhost", 8080); - when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); - - // Set up basic capabilities to ensure message contains it. Ensure it is unique from other tests - // so we catch mismatch bugs. - Capabilities.app.addAll(Capability.DAO_FULL_NODE); - } - - /** - * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state - * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers - * is not supported. - */ - private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { - KeyPair ownerKeys = TestUtils.generateKeyPair(); - - ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); - - ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); - when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); - when(stub.isValidForAddOperation()).thenReturn(true); - when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); - when(stub.getSequenceNumber()).thenReturn(1); - when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); - - return stub; - } - - static class LazyPersistableNetworkPayloadStub extends PersistableNetworkPayloadStub - implements ProcessOncePersistableNetworkPayload { - - LazyPersistableNetworkPayloadStub(byte[] hash) { - super(hash); - } - } - - // TESTCASE: GetDataResponse processing. This large tests includes all interesting variations of state - @Test - public void onMessage_GetDataResponseTest() throws NoSuchAlgorithmException { - - PersistableNetworkPayload pnp_onLocalNodeOnly = new PersistableNetworkPayloadStub(new byte[] { 1 }); - PersistableNetworkPayload pnp_inRequestOnly = new LazyPersistableNetworkPayloadStub(new byte[]{2}); - PersistableNetworkPayload pnp_onLocalNodeAndRequest = new PersistableNetworkPayloadStub(new byte[] { 3 }); - ProtectedStorageEntry pse_onLocalNodeOnly = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry pse_inRequestOnly = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry pse_onLocalNodeAndRequest = getProtectedStorageEntryForAdd(); - - this.testState.getMockedStorage().addPersistableNetworkPayload( - pnp_onLocalNodeOnly, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addPersistableNetworkPayload( - pnp_onLocalNodeAndRequest, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addProtectedStorageEntry(pse_onLocalNodeOnly, this.localNodeAddress, null, false); - this.testState.getMockedStorage().addProtectedStorageEntry(pse_onLocalNodeAndRequest, this.localNodeAddress, null, false); - - RequestDataHandler handler = new RequestDataHandler( - this.networkNode, - this.testState.getMockedStorage(), - mock(PeerManager.class), - mock(RequestDataHandler.Listener.class) - ); - - GetDataResponse getDataResponse = new GetDataResponse( - new HashSet<>(Arrays.asList(pse_inRequestOnly, pse_onLocalNodeAndRequest)), - new HashSet<>(Arrays.asList(pnp_inRequestOnly, pnp_onLocalNodeAndRequest)), - handler.nonce, false); - - NodeAddress peersNodeAddress = new NodeAddress("peer", 10); - - // Make a request with the sole reason to set the peersNodeAddress - SettableFuture sendFuture = mock(SettableFuture.class); - when(networkNode.sendMessage(any(NodeAddress.class), any(GetDataRequest.class))).thenReturn(sendFuture); - handler.requestData(peersNodeAddress, true); - - Connection connection = mock(Connection.class); - when(connection.getPeersNodeAddressOptional()).thenReturn(Optional.of(peersNodeAddress)); - handler.onMessage(getDataResponse, connection); - - Assert.assertEquals(3, this.testState.getMockedStorage().getMap().size()); - Assert.assertTrue(this.testState.getMockedStorage().getAppendOnlyDataStoreMap().containsValue(pnp_onLocalNodeOnly)); - Assert.assertTrue(this.testState.getMockedStorage().getAppendOnlyDataStoreMap().containsValue(pnp_inRequestOnly)); - Assert.assertTrue(this.testState.getMockedStorage().getAppendOnlyDataStoreMap().containsValue(pnp_onLocalNodeAndRequest)); - - Assert.assertEquals(3, this.testState.getMockedStorage().getMap().size()); - Assert.assertTrue(this.testState.getMockedStorage().getMap().containsValue(pse_onLocalNodeOnly)); - Assert.assertTrue(this.testState.getMockedStorage().getMap().containsValue(pse_inRequestOnly)); - Assert.assertTrue(this.testState.getMockedStorage().getMap().containsValue(pse_onLocalNodeAndRequest)); - } -} diff --git a/p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java b/p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java deleted file mode 100644 index e757ce07f04..00000000000 --- a/p2p/src/test/java/bisq/network/p2p/peers/getdata/RequestDataHandlerRequestDataTest.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.network.p2p.peers.getdata; - -import bisq.network.p2p.NodeAddress; -import bisq.network.p2p.TestUtils; -import bisq.network.p2p.network.Connection; -import bisq.network.p2p.network.NetworkNode; -import bisq.network.p2p.peers.PeerManager; -import bisq.network.p2p.peers.getdata.messages.GetDataRequest; -import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; -import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; -import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.TestState; -import bisq.network.p2p.storage.mocks.PersistableNetworkPayloadStub; -import bisq.network.p2p.storage.mocks.ProtectedStoragePayloadStub; -import bisq.network.p2p.storage.payload.PersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStorageEntry; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.app.Capabilities; -import bisq.common.app.Capability; - -import com.google.common.util.concurrent.SettableFuture; - -import java.security.KeyPair; -import java.security.NoSuchAlgorithmException; - -import java.util.Set; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - - - -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import static org.mockito.Mockito.*; - -public class RequestDataHandlerRequestDataTest { - private TestState testState; - - @Mock - NetworkNode networkNode; - - private NodeAddress localNodeAddress; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - this.testState = new TestState(); - - this.localNodeAddress = new NodeAddress("localhost", 8080); - when(networkNode.getNodeAddress()).thenReturn(this.localNodeAddress); - - // Set up basic capabilities to ensure message contains it - Capabilities.app.addAll(Capability.MEDIATION); - } - - /** - * Returns true if the target bytes are found in the container set. - */ - private boolean byteSetContains(Set container, byte[] target) { - // Set.contains() doesn't do a deep compare, so generate a Set so equals() does what - // we want - Set translatedContainer = - P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(container); - - return translatedContainer.contains(new P2PDataStorage.ByteArray(target)); - } - - /** - * Generates a unique ProtectedStorageEntry that is valid for add. This is used to initialize P2PDataStorage state - * so the tests can validate the correct behavior. Adds of identical payloads with different sequence numbers - * is not supported. - */ - private ProtectedStorageEntry getProtectedStorageEntryForAdd() throws NoSuchAlgorithmException { - KeyPair ownerKeys = TestUtils.generateKeyPair(); - - ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); - - ProtectedStorageEntry stub = mock(ProtectedStorageEntry.class); - when(stub.getOwnerPubKey()).thenReturn(ownerKeys.getPublic()); - when(stub.isValidForAddOperation()).thenReturn(true); - when(stub.matchesRelevantPubKey(any(ProtectedStorageEntry.class))).thenReturn(true); - when(stub.getSequenceNumber()).thenReturn(1); - when(stub.getProtectedStoragePayload()).thenReturn(protectedStoragePayload); - - return stub; - } - - // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest - @Test - public void requestData_EmptyP2PDataStore_PreliminaryGetDataRequest() { - SettableFuture sendFuture = mock(SettableFuture.class); - - final ArgumentCaptor captor = ArgumentCaptor.forClass(PreliminaryGetDataRequest.class); - when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); - - RequestDataHandler handler = new RequestDataHandler( - this.networkNode, - this.testState.getMockedStorage(), - mock(PeerManager.class), - mock(RequestDataHandler.Listener.class) - ); - - handler.requestData(this.localNodeAddress, true); - - // expect empty message since p2pDataStore is empty - PreliminaryGetDataRequest getDataRequest = captor.getValue(); - - Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); - Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); - Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); - } - - // TESTCASE: P2PDataStorage with no entries returns an empty PreliminaryGetDataRequest - @Test - public void requestData_EmptyP2PDataStore_GetUpdatedDataRequest() { - SettableFuture sendFuture = mock(SettableFuture.class); - - final ArgumentCaptor captor = ArgumentCaptor.forClass(GetUpdatedDataRequest.class); - when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); - - RequestDataHandler handler = new RequestDataHandler( - this.networkNode, - this.testState.getMockedStorage(), - mock(PeerManager.class), - mock(RequestDataHandler.Listener.class) - ); - - handler.requestData(this.localNodeAddress, false); - - // expect empty message since p2pDataStore is empty - GetUpdatedDataRequest getDataRequest = captor.getValue(); - - Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); - Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); - Assert.assertTrue(getDataRequest.getExcludedKeys().isEmpty()); - } - - // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates - // correct GetDataRequestMessage with both sets of keys. - @Test - public void requestData_FilledP2PDataStore_PreliminaryGetDataRequest() throws NoSuchAlgorithmException { - SettableFuture sendFuture = mock(SettableFuture.class); - - PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); - PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); - ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); - - this.testState.getMockedStorage().addPersistableNetworkPayload( - toAdd1, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addPersistableNetworkPayload( - toAdd2, this.localNodeAddress, false, false, false, false); - - this.testState.getMockedStorage().addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); - this.testState.getMockedStorage().addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); - - final ArgumentCaptor captor = ArgumentCaptor.forClass(PreliminaryGetDataRequest.class); - when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); - - RequestDataHandler handler = new RequestDataHandler( - this.networkNode, - this.testState.getMockedStorage(), - mock(PeerManager.class), - mock(RequestDataHandler.Listener.class) - ); - - handler.requestData(this.localNodeAddress, true); - - // expect empty message since p2pDataStore is empty - PreliminaryGetDataRequest getDataRequest = captor.getValue(); - - Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); - Assert.assertEquals(getDataRequest.getSupportedCapabilities(), Capabilities.app); - Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), - P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), - P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); - } - - // TESTCASE: P2PDataStorage with PersistableNetworkPayloads and ProtectedStorageEntry generates - // correct GetDataRequestMessage with both sets of keys. - @Test - public void requestData_FilledP2PDataStore_GetUpdatedDataRequest() throws NoSuchAlgorithmException { - SettableFuture sendFuture = mock(SettableFuture.class); - - PersistableNetworkPayload toAdd1 = new PersistableNetworkPayloadStub(new byte[] { 1 }); - PersistableNetworkPayload toAdd2 = new PersistableNetworkPayloadStub(new byte[] { 2 }); - ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); - ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); - - this.testState.getMockedStorage().addPersistableNetworkPayload( - toAdd1, this.localNodeAddress, false, false, false, false); - this.testState.getMockedStorage().addPersistableNetworkPayload( - toAdd2, this.localNodeAddress, false, false, false, false); - - this.testState.getMockedStorage().addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); - this.testState.getMockedStorage().addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); - - final ArgumentCaptor captor = ArgumentCaptor.forClass(GetUpdatedDataRequest.class); - when(networkNode.sendMessage(any(NodeAddress.class), captor.capture())).thenReturn(sendFuture); - - RequestDataHandler handler = new RequestDataHandler( - this.networkNode, - this.testState.getMockedStorage(), - mock(PeerManager.class), - mock(RequestDataHandler.Listener.class) - ); - - handler.requestData(this.localNodeAddress, false); - - // expect empty message since p2pDataStore is empty - GetUpdatedDataRequest getDataRequest = captor.getValue(); - - Assert.assertEquals(getDataRequest.getNonce(), handler.nonce); - Assert.assertEquals(getDataRequest.getSenderNodeAddress(), this.localNodeAddress); - Assert.assertEquals(4, getDataRequest.getExcludedKeys().size()); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd1.getHash())); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), toAdd2.getHash())); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), - P2PDataStorage.get32ByteHash(toAdd3.getProtectedStoragePayload()))); - Assert.assertTrue(byteSetContains(getDataRequest.getExcludedKeys(), - P2PDataStorage.get32ByteHash(toAdd4.getProtectedStoragePayload()))); - } -} diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index 15e80a9c6ac..51d831df41a 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -51,8 +51,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import lombok.Getter; - import org.junit.Assert; import org.mockito.ArgumentCaptor; @@ -67,7 +65,6 @@ public class TestState { static final int MAX_SEQUENCE_NUMBER_MAP_SIZE_BEFORE_PURGE = 5; - @Getter P2PDataStorage mockedStorage; final Broadcaster mockBroadcaster; @@ -77,7 +74,7 @@ public class TestState { private final ProtectedDataStoreService protectedDataStoreService; final ClockFake clockFake; - public TestState() { + TestState() { this.mockBroadcaster = mock(Broadcaster.class); this.mockSeqNrStorage = mock(Storage.class); this.clockFake = new ClockFake(); From 06493235055d0fc2aefac59fa3ca0e1e4a2e4a23 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 23 Nov 2019 09:54:20 -0800 Subject: [PATCH 29/51] Make addPersistableNetworkPayloadFromInitialRequest private Now that the only user is internal, the API can be made private and the tests can be removed. This involved adding a few test cases to processGetDataResponse to ensure the invalid hash size condition was still covered. --- .../network/p2p/storage/P2PDataStorage.java | 4 +-- ...aStoragePersistableNetworkPayloadTest.java | 28 +++++-------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 551b15bae49..c0a0dacf657 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -542,15 +542,13 @@ public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, // Overwriting an entry would be also no issue. We also skip notifying listeners as we get called before the domain // is ready so no listeners are set anyway. We might get called twice from a redundant call later, so listeners // might be added then but as we have the data already added calling them would be irrelevant as well. - public boolean addPersistableNetworkPayloadFromInitialRequest(PersistableNetworkPayload payload) { + private void addPersistableNetworkPayloadFromInitialRequest(PersistableNetworkPayload payload) { byte[] hash = payload.getHash(); if (payload.verifyHashSize()) { ByteArray hashAsByteArray = new ByteArray(hash); appendOnlyDataStoreService.put(hashAsByteArray, payload); - return true; } else { log.warn("We got a hash exceeding our permitted size"); - return false; } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java index 45c1ce17689..ca5e2492e29 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java @@ -49,11 +49,10 @@ * Each subclass (Payload type) can optionally add additional tests that verify functionality only relevant * to that payload. * - * Each test case is run through 4 entry points to verify the correct behavior: + * Each test case is run through 3 entry points to verify the correct behavior: * - * 1. RequestData path [addPersistableNetworkPayloadFromInitialRequest] - * 2 & 3 Client API [addPersistableNetworkPayload(reBroadcast=(true && false))] - * 4. onMessage() [onMessage(AddPersistableNetworkPayloadMessage)] + * 1 & 2 Client API [addPersistableNetworkPayload(reBroadcast=(true && false))] + * 3. onMessage() [onMessage(AddPersistableNetworkPayloadMessage)] */ @SuppressWarnings("unused") public class P2PDataStoragePersistableNetworkPayloadTest { @@ -81,11 +80,6 @@ public abstract static class AddPersistableNetworkPayloadTest { enum TestCase { PUBLIC_API, ON_MESSAGE, - INIT, - } - - boolean expectBroadcastOnStateChange() { - return this.testCase != TestCase.INIT; } boolean expectedIsDataOwner() { @@ -95,9 +89,7 @@ boolean expectedIsDataOwner() { void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean expectedReturnValue, boolean expectedStateChange) { SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); - if (this.testCase == TestCase.INIT) { - Assert.assertEquals(expectedReturnValue, this.testState.mockedStorage.addPersistableNetworkPayloadFromInitialRequest(persistableNetworkPayload)); - } else if (this.testCase == TestCase.PUBLIC_API) { + if (this.testCase == TestCase.PUBLIC_API) { Assert.assertEquals(expectedReturnValue, this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, TestState.getTestNodeAddress(), true, this.allowBroadcast, this.reBroadcast, this.checkDate)); } else { // onMessage @@ -107,9 +99,7 @@ void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean testState.mockedStorage.onMessage(new AddPersistableNetworkPayloadMessage(persistableNetworkPayload), mockedConnection); } - boolean expectedBroadcast = expectedStateChange && this.expectBroadcastOnStateChange(); - - this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedBroadcast, expectedBroadcast, this.expectedIsDataOwner()); + this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedStateChange, expectedStateChange, this.expectedIsDataOwner()); } @Before @@ -123,9 +113,6 @@ public void setup() { public static Collection data() { List data = new ArrayList<>(); - // Init doesn't use other parameters - data.add(new Object[] { TestCase.INIT, false, false, false }); - // onMessage doesn't use other parameters data.add(new Object[] { TestCase.ON_MESSAGE, false, false, false }); @@ -149,9 +136,8 @@ public void addPersistableNetworkPayload() { public void addPersistableNetworkPayloadDuplicate() { doAddAndVerify(this.persistableNetworkPayload, true, true); - // Second call only succeeds if reBroadcast was set or we are adding through the init - // path which just overwrites - boolean expectedReturnValue = this.reBroadcast || this.testCase == TestCase.INIT; + // Second call only succeeds if reBroadcast was set + boolean expectedReturnValue = this.reBroadcast; doAddAndVerify(this.persistableNetworkPayload, expectedReturnValue, false); } } From 56a7661a0273048ad0dcb4cc2cad77d6ddf69b3a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 16:50:05 -0800 Subject: [PATCH 30/51] [REFACTOR] Clean up ClientAPI for addPersistableNetworkPayload Now that more callers have moved internal, the public facing API can be cleaner and more simple. This should lead to a more maintainable API and less sharp edges with future development work. --- .../java/bisq/network/p2p/P2PService.java | 2 +- .../network/p2p/storage/P2PDataStorage.java | 27 +++++++++++++++---- ...2PDataStorageBuildGetDataResponseTest.java | 12 ++++----- ...aStoragePersistableNetworkPayloadTest.java | 14 +++------- .../P2PDataStorageProcessGetDataResponse.java | 4 +-- .../P2PDataStorageRemoveExpiredTest.java | 2 +- .../P2PDataStorageRequestDataTest.java | 12 +++------ 7 files changed, 40 insertions(+), 33 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 733cc385cce..8b6d7cdbd3c 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -779,7 +779,7 @@ private void delayedRemoveEntryFromMailbox(DecryptedMessageWithPubKey decryptedM /////////////////////////////////////////////////////////////////////////////////////////// public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, boolean reBroadcast) { - return p2PDataStorage.addPersistableNetworkPayload(payload, networkNode.getNodeAddress(), true, true, reBroadcast, false); + return p2PDataStorage.addPersistableNetworkPayload(payload, networkNode.getNodeAddress(), reBroadcast); } public boolean addProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, boolean isDataOwner) { diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index c0a0dacf657..4d5f2c50a82 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -490,15 +490,32 @@ public void onError(Throwable throwable) { /////////////////////////////////////////////////////////////////////////////////////////// - // API + // Client API /////////////////////////////////////////////////////////////////////////////////////////// + /** + * Adds a PersistableNetworkPayload to the local P2P data storage. If it does not already exist locally, it will + * be broadcast to the P2P network. + * @param payload PersistableNetworkPayload to add to the network + * @param sender local NodeAddress, if available + * @param allowReBroadcast true if the PersistableNetworkPayload should be rebroadcast even if it + * already exists locally + * @return true if the PersistableNetworkPayload passes all validation and exists in the P2PDataStore + * on completion + */ public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, @Nullable NodeAddress sender, - boolean isDataOwner, - boolean allowBroadcast, - boolean reBroadcast, - boolean checkDate) { + boolean allowReBroadcast) { + return addPersistableNetworkPayload( + payload, sender, true, true, allowReBroadcast, false); + } + + private boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, + @Nullable NodeAddress sender, + boolean isDataOwner, + boolean allowBroadcast, + boolean reBroadcast, + boolean checkDate) { log.trace("addPersistableNetworkPayload payload={}", payload); // Payload hash size does not match expectation for that type of message. diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java index 173d3358041..19dbd6dd3a3 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java @@ -166,7 +166,7 @@ public void buildGetDataResponse_knownPNPDoNothing() { PersistableNetworkPayload fromPeerAndLocal = new PersistableNetworkPayloadStub(new byte[]{1}); this.testState.mockedStorage.addPersistableNetworkPayload( - fromPeerAndLocal, this.localNodeAddress, false, false, false, false); + fromPeerAndLocal, this.localNodeAddress, false); GetDataRequest getDataRequest = this.buildGetDataRequest( @@ -194,7 +194,7 @@ public void buildGetDataResponse_unknownPNPSendBack() { PersistableNetworkPayload onlyLocal = new PersistableNetworkPayloadStub(new byte[]{1}); this.testState.mockedStorage.addPersistableNetworkPayload( - onlyLocal, this.localNodeAddress, false, false, false, false); + onlyLocal, this.localNodeAddress, false); GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); @@ -221,9 +221,9 @@ public void buildGetDataResponse_unknownPNPSendBackTruncation() { PersistableNetworkPayload onlyLocal2 = new PersistableNetworkPayloadStub(new byte[]{2}); this.testState.mockedStorage.addPersistableNetworkPayload( - onlyLocal1, this.localNodeAddress, false, false, false, false); + onlyLocal1, this.localNodeAddress, false); this.testState.mockedStorage.addPersistableNetworkPayload( - onlyLocal2, this.localNodeAddress, false, false, false, false); + onlyLocal2, this.localNodeAddress, false); GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); @@ -252,7 +252,7 @@ public void buildGetDataResponse_unknownPNPCapabilitiesMismatchDontSendBack() { new byte[]{1}); this.testState.mockedStorage.addPersistableNetworkPayload( - onlyLocal, this.localNodeAddress, false, false, false, false); + onlyLocal, this.localNodeAddress, false); GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); @@ -280,7 +280,7 @@ public void buildGetDataResponse_unknownPNPCapabilitiesMatch() { new byte[]{1}); this.testState.mockedStorage.addPersistableNetworkPayload( - onlyLocal, this.localNodeAddress, false, false, false, false); + onlyLocal, this.localNodeAddress, false); GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java index ca5e2492e29..d0042df1f2f 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java @@ -65,14 +65,8 @@ public abstract static class AddPersistableNetworkPayloadTest { public TestCase testCase; @Parameterized.Parameter(1) - public boolean allowBroadcast; - - @Parameterized.Parameter(2) public boolean reBroadcast; - @Parameterized.Parameter(3) - public boolean checkDate; - PersistableNetworkPayload persistableNetworkPayload; abstract PersistableNetworkPayload createInstance(); @@ -91,7 +85,7 @@ void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean if (this.testCase == TestCase.PUBLIC_API) { Assert.assertEquals(expectedReturnValue, - this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, TestState.getTestNodeAddress(), true, this.allowBroadcast, this.reBroadcast, this.checkDate)); + this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, TestState.getTestNodeAddress(), this.reBroadcast)); } else { // onMessage Connection mockedConnection = mock(Connection.class); when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(TestState.getTestNodeAddress())); @@ -114,14 +108,14 @@ public static Collection data() { List data = new ArrayList<>(); // onMessage doesn't use other parameters - data.add(new Object[] { TestCase.ON_MESSAGE, false, false, false }); + data.add(new Object[] { TestCase.ON_MESSAGE, false }); // Client API uses two permutations // Normal path - data.add(new Object[] { TestCase.PUBLIC_API, true, true, false }); + data.add(new Object[] { TestCase.PUBLIC_API, true }); // Refresh path - data.add(new Object[] { TestCase.PUBLIC_API, true, false, false }); + data.add(new Object[] { TestCase.PUBLIC_API, false }); return data; } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java index c7177b005d7..33cb364b3e1 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -138,7 +138,7 @@ public void processGetDataResponse_newInvalidPNPDoesNothing() { public void processGetDataResponse_duplicatePNPDoesNothing() { PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(new byte[] { 1 }); this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, - this.peerNodeAddress, false, false, false, false); + this.peerNodeAddress, false); GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); @@ -166,7 +166,7 @@ public void processGetDataResponse_newPNPUpdatesState_LazyProcessed() { public void processGetDataResponse_duplicatePNPDoesNothing_LazyProcessed() { PersistableNetworkPayload persistableNetworkPayload = new LazyPersistableNetworkPayloadStub(new byte[] { 1 }); this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload, - this.peerNodeAddress, false, false, false, false); + this.peerNodeAddress, false); GetDataResponse getDataResponse = buildGetDataResponse(persistableNetworkPayload); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java index 454799ed515..440d4b1c42d 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java @@ -75,7 +75,7 @@ public void removeExpiredEntries_SkipsNonExpirableEntries() throws NoSuchAlgorit public void removeExpiredEntries_skipsPersistableNetworkPayload() { PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(true); - Assert.assertTrue(this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload,getTestNodeAddress(), true, true, false, false)); + Assert.assertTrue(this.testState.mockedStorage.addPersistableNetworkPayload(persistableNetworkPayload,getTestNodeAddress(), false)); this.testState.mockedStorage.removeExpiredEntries(); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java index 55d16344d64..6fee8212b50 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java @@ -121,10 +121,8 @@ public void buildPreliminaryGetDataRequest_FilledP2PDataStore() throws NoSuchAlg ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); - this.testState.mockedStorage.addPersistableNetworkPayload( - toAdd1, this.localNodeAddress, false, false, false, false); - this.testState.mockedStorage.addPersistableNetworkPayload( - toAdd2, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload(toAdd1, this.localNodeAddress, false); + this.testState.mockedStorage.addPersistableNetworkPayload(toAdd2, this.localNodeAddress, false); this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); @@ -151,10 +149,8 @@ public void requestData_FilledP2PDataStore_GetUpdatedDataRequest() throws NoSuch ProtectedStorageEntry toAdd3 = getProtectedStorageEntryForAdd(); ProtectedStorageEntry toAdd4 = getProtectedStorageEntryForAdd(); - this.testState.mockedStorage.addPersistableNetworkPayload( - toAdd1, this.localNodeAddress, false, false, false, false); - this.testState.mockedStorage.addPersistableNetworkPayload( - toAdd2, this.localNodeAddress, false, false, false, false); + this.testState.mockedStorage.addPersistableNetworkPayload(toAdd1, this.localNodeAddress, false); + this.testState.mockedStorage.addPersistableNetworkPayload(toAdd2, this.localNodeAddress, false); this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); From 0e6b1a204401024387d106b783c731d322ff10c9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 17:07:35 -0800 Subject: [PATCH 31/51] [REFACTOR] Clean up ClientAPI for addProtectedStorageEntry Remove isDataOwner from the client API. All users pass in true. All test users don't care. --- .../main/java/bisq/core/alert/AlertManager.java | 2 +- .../proposal/MyProposalListService.java | 2 +- .../java/bisq/core/filter/FilterManager.java | 2 +- .../java/bisq/core/offer/OfferBookService.java | 2 +- .../dispute/agent/DisputeAgentService.java | 2 +- .../main/java/bisq/network/p2p/P2PService.java | 6 +++--- .../network/p2p/storage/P2PDataStorage.java | 17 +++++++++++++---- .../P2PDataStorageBuildGetDataResponseTest.java | 12 ++++++------ .../storage/P2PDataStorageClientAPITest.java | 14 +++++++------- .../P2PDataStorageGetDataIntegrationTest.java | 14 +++++++------- .../P2PDataStorageProcessGetDataResponse.java | 2 +- ...P2PDataStorageProtectedStorageEntryTest.java | 4 +--- .../P2PDataStorageRemoveExpiredTest.java | 16 ++++++++-------- .../storage/P2PDataStorageRequestDataTest.java | 8 ++++---- .../p2p/storage/P2PDataStoreDisconnectTest.java | 4 ++-- 15 files changed, 57 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/bisq/core/alert/AlertManager.java b/core/src/main/java/bisq/core/alert/AlertManager.java index 658c675ee32..0887a7556eb 100644 --- a/core/src/main/java/bisq/core/alert/AlertManager.java +++ b/core/src/main/java/bisq/core/alert/AlertManager.java @@ -127,7 +127,7 @@ public boolean addAlertMessageIfKeyIsValid(Alert alert, String privKeyString) { if (isKeyValid) { signAndAddSignatureToAlertMessage(alert); user.setDevelopersAlert(alert); - boolean result = p2PService.addProtectedStorageEntry(alert, true); + boolean result = p2PService.addProtectedStorageEntry(alert); if (result) { log.trace("Add alertMessage to network was successful. AlertMessage={}", alert); } diff --git a/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java b/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java index d0933144c56..e49652919ea 100644 --- a/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java +++ b/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java @@ -214,7 +214,7 @@ private void addToP2PNetworkAsProtectedData(Proposal proposal, ErrorMessageHandl } private boolean addToP2PNetworkAsProtectedData(Proposal proposal) { - return p2PService.addProtectedStorageEntry(new TempProposalPayload(proposal, signaturePubKey), true); + return p2PService.addProtectedStorageEntry(new TempProposalPayload(proposal, signaturePubKey)); } private void rePublishMyProposalsOnceWellConnected() { diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index e14f7baa613..b85a46717c1 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -265,7 +265,7 @@ public boolean addFilterMessageIfKeyIsValid(Filter filter, String privKeyString) signAndAddSignatureToFilter(filter); user.setDevelopersFilter(filter); - boolean result = p2PService.addProtectedStorageEntry(filter, true); + boolean result = p2PService.addProtectedStorageEntry(filter); if (result) log.trace("Add filter to network was successful. FilterMessage = {}", filter); diff --git a/core/src/main/java/bisq/core/offer/OfferBookService.java b/core/src/main/java/bisq/core/offer/OfferBookService.java index 6ccaa4dfa3e..55e20478ab7 100644 --- a/core/src/main/java/bisq/core/offer/OfferBookService.java +++ b/core/src/main/java/bisq/core/offer/OfferBookService.java @@ -148,7 +148,7 @@ public void addOffer(Offer offer, ResultHandler resultHandler, ErrorMessageHandl return; } - boolean result = p2PService.addProtectedStorageEntry(offer.getOfferPayload(), true); + boolean result = p2PService.addProtectedStorageEntry(offer.getOfferPayload()); if (result) { resultHandler.handleResult(); } else { diff --git a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java index 2a5c284f2b4..cfaafd66c19 100644 --- a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java +++ b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java @@ -64,7 +64,7 @@ public void addDisputeAgent(T disputeAgent, log.debug("addDisputeAgent disputeAgent.hashCode() " + disputeAgent.hashCode()); if (!BisqEnvironment.getBaseCurrencyNetwork().isMainnet() || !Utilities.encodeToHex(disputeAgent.getRegistrationPubKey()).equals(DevEnv.DEV_PRIVILEGE_PUB_KEY)) { - boolean result = p2PService.addProtectedStorageEntry(disputeAgent, true); + boolean result = p2PService.addProtectedStorageEntry(disputeAgent); if (result) { log.trace("Add disputeAgent to network was successful. DisputeAgent.hashCode() = " + disputeAgent.hashCode()); resultHandler.handleResult(); diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 8b6d7cdbd3c..bb51b83f6f8 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -699,7 +699,7 @@ public void onBroadcastFailed(String errorMessage) { // to the logic from BroadcastHandler.sendToPeer } }; - boolean result = p2PDataStorage.addProtectedStorageEntry(protectedMailboxStorageEntry, networkNode.getNodeAddress(), listener, true); + boolean result = p2PDataStorage.addProtectedStorageEntry(protectedMailboxStorageEntry, networkNode.getNodeAddress(), listener); if (!result) { sendMailboxMessageListener.onFault("Data already exists in our local database"); @@ -782,11 +782,11 @@ public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, b return p2PDataStorage.addPersistableNetworkPayload(payload, networkNode.getNodeAddress(), reBroadcast); } - public boolean addProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, boolean isDataOwner) { + public boolean addProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload) { if (isBootstrapped()) { try { ProtectedStorageEntry protectedStorageEntry = p2PDataStorage.getProtectedStorageEntry(protectedStoragePayload, keyRing.getSignatureKeyPair()); - return p2PDataStorage.addProtectedStorageEntry(protectedStorageEntry, networkNode.getNodeAddress(), null, isDataOwner); + return p2PDataStorage.addProtectedStorageEntry(protectedStorageEntry, networkNode.getNodeAddress(), null); } catch (CryptoException e) { log.error("Signing at getDataWithSignedSeqNr failed. That should never happen."); return false; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 4d5f2c50a82..97c23efe803 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -415,7 +415,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof BroadcastMessage) { connection.getPeersNodeAddressOptional().ifPresent(peersNodeAddress -> { if (networkEnvelope instanceof AddDataMessage) { - addProtectedStorageEntry(((AddDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, null, false); + addProtectedStorageEntry(((AddDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, null, false, true); } else if (networkEnvelope instanceof RemoveDataMessage) { remove(((RemoveDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, false); } else if (networkEnvelope instanceof RemoveMailboxDataMessage) { @@ -569,12 +569,21 @@ private void addPersistableNetworkPayloadFromInitialRequest(PersistableNetworkPa } } + /** + * Adds a ProtectedStorageEntry to the local P2P data storage. If it does not already exist locally, it will be + * broadcast to the P2P network. + * + * @param protectedStorageEntry ProtectedStorageEntry to add to the network + * @param sender local NodeAddress, if available + * @param listener optional listener that can be used to receive events on broadcast + * @return true if the ProtectedStorageEntry was added to the local P2P data storage and broadcast + */ public boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, - @Nullable BroadcastHandler.Listener listener, boolean isDataOwner) { - return addProtectedStorageEntry(protectedStorageEntry, sender, listener, isDataOwner, true); + @Nullable BroadcastHandler.Listener listener) { + return addProtectedStorageEntry(protectedStorageEntry, sender, listener, true, true); } - public boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, + private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener listener, boolean isDataOwner, diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java index 19dbd6dd3a3..5a65f435c45 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageBuildGetDataResponseTest.java @@ -336,7 +336,7 @@ public void buildGetDataResponse_knownPSEDoNothing() throws NoSuchAlgorithmExcep P2PDataStorage.get32ByteHash(fromPeerAndLocal.getProtectedStoragePayload())))); this.testState.mockedStorage.addProtectedStorageEntry( - fromPeerAndLocal, this.localNodeAddress, null, false); + fromPeerAndLocal, this.localNodeAddress, null); AtomicBoolean outPNPTruncated = new AtomicBoolean(false); AtomicBoolean outPSETruncated = new AtomicBoolean(false); @@ -361,7 +361,7 @@ public void buildGetDataResponse_unknownPSESendBack() throws NoSuchAlgorithmExce GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); this.testState.mockedStorage.addProtectedStorageEntry( - onlyLocal, this.localNodeAddress, null, false); + onlyLocal, this.localNodeAddress, null); AtomicBoolean outPNPTruncated = new AtomicBoolean(false); AtomicBoolean outPSETruncated = new AtomicBoolean(false); @@ -387,9 +387,9 @@ public void buildGetDataResponse_unknownPSESendBackTruncation() throws NoSuchAlg GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); this.testState.mockedStorage.addProtectedStorageEntry( - onlyLocal1, this.localNodeAddress, null, false); + onlyLocal1, this.localNodeAddress, null); this.testState.mockedStorage.addProtectedStorageEntry( - onlyLocal2, this.localNodeAddress, null, false); + onlyLocal2, this.localNodeAddress, null); AtomicBoolean outPNPTruncated = new AtomicBoolean(false); AtomicBoolean outPSETruncated = new AtomicBoolean(false); @@ -416,7 +416,7 @@ public void buildGetDataResponse_unknownPSECapabilitiesMismatchDontSendBack() th getProtectedStorageEntryForAdd(new Capabilities(Collections.singletonList(Capability.MEDIATION))); this.testState.mockedStorage.addProtectedStorageEntry( - onlyLocal, this.localNodeAddress, null, false); + onlyLocal, this.localNodeAddress, null); GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); @@ -442,7 +442,7 @@ public void buildGetDataResponse_unknownPSECapabilitiesMatch() throws NoSuchAlgo getProtectedStorageEntryForAdd(new Capabilities(Collections.singletonList(Capability.MEDIATION))); this.testState.mockedStorage.addProtectedStorageEntry( - onlyLocal, this.localNodeAddress, null, false); + onlyLocal, this.localNodeAddress, null); GetDataRequest getDataRequest = this.buildGetDataRequest(1, new HashSet<>()); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index 7126b61770a..fd36838f503 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -71,7 +71,7 @@ public void getProtectedStorageEntry_NoExist() throws NoSuchAlgorithmException, ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } @@ -84,11 +84,11 @@ public void getProtectedStorageEntry() throws NoSuchAlgorithmException, CryptoEx ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true); + this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } @@ -108,7 +108,7 @@ public void getProtectedStorageEntry_FirstOnMessageSecondAPI() throws NoSuchAlgo SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); } @@ -135,7 +135,7 @@ public void getRefreshTTLMessage() throws NoSuchAlgorithmException, CryptoExcept ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true); + this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); RefreshOfferMessage refreshOfferMessage = this.testState.mockedStorage.getRefreshTTLMessage(protectedStoragePayload, ownerKeys); this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress(), true); @@ -157,7 +157,7 @@ public void getRefreshTTLMessage_FirstOnMessageSecondAPI() throws NoSuchAlgorith ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true); + this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); Connection mockedConnection = mock(Connection.class); when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(TestState.getTestNodeAddress())); @@ -202,7 +202,7 @@ public void getMailboxDataWithSignedSeqNr_AddThenRemove() throws NoSuchAlgorithm ProtectedMailboxStorageEntry protectedMailboxStorageEntry = this.testState.mockedStorage.getMailboxDataWithSignedSeqNr(mailboxStoragePayload, senderKeys, receiverKeys.getPublic()); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedMailboxStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedMailboxStorageEntry, TestState.getTestNodeAddress(), null)); protectedMailboxStorageEntry = this.testState.mockedStorage.getMailboxDataWithSignedSeqNr(mailboxStoragePayload, receiverKeys, receiverKeys.getPublic()); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java index 3b40575c633..472d65d7ea9 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java @@ -77,7 +77,7 @@ public void basicSynchronizationWorks() throws NoSuchAlgorithmException { P2PDataStorage clientNode = clientNodeTestState.mockedStorage; ProtectedStorageEntry onSeedNode = getProtectedStorageEntry(); - seedNode.addProtectedStorageEntry(onSeedNode, null, null, false); + seedNode.addProtectedStorageEntry(onSeedNode, null, null); GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); @@ -102,9 +102,9 @@ public void basicSynchronizationWorksAfterRestartTransient() throws NoSuchAlgori TestState clientNodeTestState = new TestState(); P2PDataStorage clientNode = clientNodeTestState.mockedStorage; - seedNode.addProtectedStorageEntry(transientEntry, null, null, false); + seedNode.addProtectedStorageEntry(transientEntry, null, null); - clientNode.addProtectedStorageEntry(transientEntry, null, null, false); + clientNode.addProtectedStorageEntry(transientEntry, null, null); clientNodeTestState.simulateRestart(); clientNode = clientNodeTestState.mockedStorage; @@ -135,9 +135,9 @@ public void basicSynchronizationWorksAfterRestartPersistent() throws NoSuchAlgor TestState clientNodeTestState = new TestState(); P2PDataStorage clientNode = clientNodeTestState.mockedStorage; - seedNode.addProtectedStorageEntry(persistentEntry, null, null, false); + seedNode.addProtectedStorageEntry(persistentEntry, null, null); - clientNode.addProtectedStorageEntry(persistentEntry, null, null, false); + clientNode.addProtectedStorageEntry(persistentEntry, null, null); clientNodeTestState.simulateRestart(); clientNode = clientNodeTestState.mockedStorage; @@ -171,8 +171,8 @@ public void lostRemoveNeverUpdated() throws NoSuchAlgorithmException { ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry onSeedNodeAndClientNode = getProtectedStorageEntry( ownerKeys.getPublic(), protectedStoragePayload, 1); - seedNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null, false); - clientNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null, false); + seedNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null); + clientNode.addProtectedStorageEntry(onSeedNodeAndClientNode, null, null); // Seed node sees the remove, but client node does not seedNode.remove(getProtectedStorageEntry( diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java index 33cb364b3e1..abf605ae1a6 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -231,7 +231,7 @@ public void processGetDataResponse_newPSEUpdatesState() throws NoSuchAlgorithmEx @Test public void processGetDataResponse_duplicatePSEDoesNothing() throws NoSuchAlgorithmException { ProtectedStorageEntry protectedStorageEntry = getProtectedStorageEntryForAdd(); - this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, this.peerNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, this.peerNodeAddress, null); GetDataResponse getDataResponse = buildGetDataResponse(protectedStorageEntry); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java index 04eb291f309..cd1c27be1b6 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java @@ -133,10 +133,8 @@ boolean doAdd(ProtectedStorageEntry protectedStorageEntry) { return true; } else { - // XXX: All external callers just pass in true for isDataOwner and allowBroadcast a future patch can - // remove the argument. return this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, - TestState.getTestNodeAddress(), null, true); + TestState.getTestNodeAddress(), null); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java index 440d4b1c42d..f561733e375 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java @@ -62,7 +62,7 @@ public void removeExpiredEntries_SkipsNonExpirableEntries() throws NoSuchAlgorit KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); @@ -88,7 +88,7 @@ public void removeExpiredEntries_SkipNonExpiredExpirableEntries() throws CryptoE KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); @@ -102,7 +102,7 @@ public void removeExpiredEntries_ExpiresExpiredExpirableEntries() throws CryptoE KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic(), 0); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); // Increment the clock by an hour which will cause the Payloads to be outside the TTL range this.testState.incrementClock(); @@ -119,7 +119,7 @@ public void removeExpiredEntries_SkipNonExpiredPersistableExpirableEntries() thr KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = new PersistableExpirableProtectedStoragePayloadStub(ownerKeys.getPublic()); ProtectedStorageEntry protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); @@ -133,7 +133,7 @@ public void removeExpiredEntries_ExpiresExpiredPersistableExpirableEntries() thr KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = new PersistableExpirableProtectedStoragePayloadStub(ownerKeys.getPublic(), 0); ProtectedStorageEntry protectedStorageEntry = testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); // Increment the clock by an hour which will cause the Payloads to be outside the TTL range this.testState.incrementClock(); @@ -158,14 +158,14 @@ public void removeExpiredEntries_PurgeSeqNrMap() throws CryptoException, NoSuchA ProtectedStorageEntry purgedProtectedStorageEntry = testState.mockedStorage.getProtectedStorageEntry(purgedProtectedStoragePayload, purgedOwnerKeys); expectedRemoves.add(purgedProtectedStorageEntry); - Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(purgedProtectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(purgedProtectedStorageEntry, TestState.getTestNodeAddress(), null)); for (int i = 0; i < MAX_SEQUENCE_NUMBER_MAP_SIZE_BEFORE_PURGE - 1; ++i) { KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = new PersistableExpirableProtectedStoragePayloadStub(ownerKeys.getPublic(), 0); ProtectedStorageEntry tmpEntry = testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); expectedRemoves.add(tmpEntry); - Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(tmpEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(tmpEntry, TestState.getTestNodeAddress(), null)); } // Increment the time by 5 days which is less than the purge requirement. This will allow the map to have @@ -178,7 +178,7 @@ public void removeExpiredEntries_PurgeSeqNrMap() throws CryptoException, NoSuchA ProtectedStorageEntry keepProtectedStorageEntry = testState.mockedStorage.getProtectedStorageEntry(keepProtectedStoragePayload, keepOwnerKeys); expectedRemoves.add(keepProtectedStorageEntry); - Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(keepProtectedStorageEntry, TestState.getTestNodeAddress(), null, true)); + Assert.assertTrue(testState.mockedStorage.addProtectedStorageEntry(keepProtectedStorageEntry, TestState.getTestNodeAddress(), null)); // P2PDataStorage::PURGE_AGE_DAYS == 10 days // Advance time past it so they will be valid purge targets diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java index 6fee8212b50..27e522daaad 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRequestDataTest.java @@ -124,8 +124,8 @@ public void buildPreliminaryGetDataRequest_FilledP2PDataStore() throws NoSuchAlg this.testState.mockedStorage.addPersistableNetworkPayload(toAdd1, this.localNodeAddress, false); this.testState.mockedStorage.addPersistableNetworkPayload(toAdd2, this.localNodeAddress, false); - this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); - this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null); PreliminaryGetDataRequest getDataRequest = this.testState.mockedStorage.buildPreliminaryGetDataRequest(1); @@ -152,8 +152,8 @@ public void requestData_FilledP2PDataStore_GetUpdatedDataRequest() throws NoSuch this.testState.mockedStorage.addPersistableNetworkPayload(toAdd1, this.localNodeAddress, false); this.testState.mockedStorage.addPersistableNetworkPayload(toAdd2, this.localNodeAddress, false); - this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null, false); - this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null, false); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd3, this.localNodeAddress, null); + this.testState.mockedStorage.addProtectedStorageEntry(toAdd4, this.localNodeAddress, null); GetUpdatedDataRequest getDataRequest = this.testState.mockedStorage.buildGetUpdatedDataRequest(this.localNodeAddress, 1); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java index 1f58b815188..abba32569c8 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java @@ -57,7 +57,7 @@ private static ProtectedStorageEntry populateTestState(TestState testState, ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic(), ttl); ProtectedStorageEntry protectedStorageEntry = testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null, false); + testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); return protectedStorageEntry; } @@ -186,7 +186,7 @@ private ExpirablePersistentProtectedStoragePayloadStub(PublicKey ownerPubKey) { testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); testState.mockedStorage.addProtectedStorageEntry( - protectedStorageEntry, TestState.getTestNodeAddress(), null, false); + protectedStorageEntry, TestState.getTestNodeAddress(), null); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); From 77413c9671a14d16061c5c1f8b2a9445b2bb7ffa Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 17:14:18 -0800 Subject: [PATCH 32/51] [REFACTOR] Clean up ClientAPI for remove Remove isDataOwner from the client API. All users pass in true. All test users don't care. --- .../src/main/java/bisq/core/alert/AlertManager.java | 2 +- .../governance/proposal/MyProposalListService.java | 2 +- .../main/java/bisq/core/filter/FilterManager.java | 2 +- .../main/java/bisq/core/offer/OfferBookService.java | 2 +- .../support/dispute/agent/DisputeAgentService.java | 2 +- p2p/src/main/java/bisq/network/p2p/P2PService.java | 6 +++--- .../bisq/network/p2p/storage/P2PDataStorage.java | 13 +++++++++++++ .../p2p/storage/P2PDataStorageClientAPITest.java | 6 +++--- .../P2PDataStorageGetDataIntegrationTest.java | 2 +- .../P2PDataStorageProtectedStorageEntryTest.java | 6 ++---- 10 files changed, 27 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/bisq/core/alert/AlertManager.java b/core/src/main/java/bisq/core/alert/AlertManager.java index 0887a7556eb..b21b7075054 100644 --- a/core/src/main/java/bisq/core/alert/AlertManager.java +++ b/core/src/main/java/bisq/core/alert/AlertManager.java @@ -139,7 +139,7 @@ public boolean addAlertMessageIfKeyIsValid(Alert alert, String privKeyString) { public boolean removeAlertMessageIfKeyIsValid(String privKeyString) { Alert alert = user.getDevelopersAlert(); if (isKeyValid(privKeyString) && alert != null) { - if (p2PService.removeData(alert, true)) + if (p2PService.removeData(alert)) log.trace("Remove alertMessage from network was successful. AlertMessage={}", alert); user.setDevelopersAlert(null); diff --git a/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java b/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java index e49652919ea..bad67f0bd6f 100644 --- a/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java +++ b/core/src/main/java/bisq/core/dao/governance/proposal/MyProposalListService.java @@ -163,7 +163,7 @@ public void onFailure(TxBroadcastException exception) { public boolean remove(Proposal proposal) { if (canRemoveProposal(proposal, daoStateService, periodService)) { - boolean success = p2PService.removeData(new TempProposalPayload(proposal, signaturePubKey), true); + boolean success = p2PService.removeData(new TempProposalPayload(proposal, signaturePubKey)); if (!success) log.warn("Removal of proposal from p2p network failed. proposal={}", proposal); diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index b85a46717c1..64ec22c9a1b 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -278,7 +278,7 @@ public boolean removeFilterMessageIfKeyIsValid(String privKeyString) { Filter filter = user.getDevelopersFilter(); if (filter == null) { log.warn("Developers filter is null"); - } else if (p2PService.removeData(filter, true)) { + } else if (p2PService.removeData(filter)) { log.trace("Remove filter from network was successful. FilterMessage = {}", filter); user.setDevelopersFilter(null); } else { diff --git a/core/src/main/java/bisq/core/offer/OfferBookService.java b/core/src/main/java/bisq/core/offer/OfferBookService.java index 55e20478ab7..a254f349e0b 100644 --- a/core/src/main/java/bisq/core/offer/OfferBookService.java +++ b/core/src/main/java/bisq/core/offer/OfferBookService.java @@ -187,7 +187,7 @@ public void deactivateOffer(OfferPayload offerPayload, public void removeOffer(OfferPayload offerPayload, @Nullable ResultHandler resultHandler, @Nullable ErrorMessageHandler errorMessageHandler) { - if (p2PService.removeData(offerPayload, true)) { + if (p2PService.removeData(offerPayload)) { if (resultHandler != null) resultHandler.handleResult(); } else { diff --git a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java index cfaafd66c19..e7661d8421d 100644 --- a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java +++ b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java @@ -81,7 +81,7 @@ public void removeDisputeAgent(T disputeAgent, ResultHandler resultHandler, ErrorMessageHandler errorMessageHandler) { log.debug("removeDisputeAgent disputeAgent.hashCode() " + disputeAgent.hashCode()); - if (p2PService.removeData(disputeAgent, true)) { + if (p2PService.removeData(disputeAgent)) { log.trace("Remove disputeAgent from network was successful. DisputeAgent.hashCode() = " + disputeAgent.hashCode()); resultHandler.handleResult(); } else { diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index bb51b83f6f8..d4909f07c96 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -760,7 +760,7 @@ private void delayedRemoveEntryFromMailbox(DecryptedMessageWithPubKey decryptedM expirableMailboxStoragePayload, keyRing.getSignatureKeyPair(), receiversPubKey); - p2PDataStorage.remove(protectedMailboxStorageEntry, networkNode.getNodeAddress(), true); + p2PDataStorage.remove(protectedMailboxStorageEntry, networkNode.getNodeAddress()); } catch (CryptoException e) { log.error("Signing at getDataWithSignedSeqNr failed. That should never happen."); } @@ -810,11 +810,11 @@ public boolean refreshTTL(ProtectedStoragePayload protectedStoragePayload, boole } } - public boolean removeData(ProtectedStoragePayload protectedStoragePayload, boolean isDataOwner) { + public boolean removeData(ProtectedStoragePayload protectedStoragePayload) { if (isBootstrapped()) { try { ProtectedStorageEntry protectedStorageEntry = p2PDataStorage.getProtectedStorageEntry(protectedStoragePayload, keyRing.getSignatureKeyPair()); - return p2PDataStorage.remove(protectedStorageEntry, networkNode.getNodeAddress(), isDataOwner); + return p2PDataStorage.remove(protectedStorageEntry, networkNode.getNodeAddress()); } catch (CryptoException e) { log.error("Signing at getDataWithSignedSeqNr failed. That should never happen."); return false; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 97c23efe803..f267b11b108 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -691,7 +691,20 @@ public boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, return true; } + /** + * Removes a ProtectedStorageEntry from the local P2P data storage. If it is successful, it will broadcast that + * change to the P2P network. + * + * @param protectedStorageEntry ProtectedStorageEntry to add to the network + * @param sender local NodeAddress, if available + * @return true if the ProtectedStorageEntry was removed from the local P2P data storage and broadcast + */ public boolean remove(ProtectedStorageEntry protectedStorageEntry, + @Nullable NodeAddress sender) { + return remove(protectedStorageEntry, sender, true); + } + + private boolean remove(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, boolean isDataOwner) { ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index fd36838f503..8ac4a409e61 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -186,7 +186,7 @@ public void getMailboxDataWithSignedSeqNr_RemoveNoExist() throws NoSuchAlgorithm this.testState.mockedStorage.getMailboxDataWithSignedSeqNr(mailboxStoragePayload, receiverKeys, receiverKeys.getPublic()); SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); - Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress(), true)); + Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, false, false, false, true, true); } @@ -208,7 +208,7 @@ public void getMailboxDataWithSignedSeqNr_AddThenRemove() throws NoSuchAlgorithm this.testState.mockedStorage.getMailboxDataWithSignedSeqNr(mailboxStoragePayload, receiverKeys, receiverKeys.getPublic()); SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); - Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress(), true)); + Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, true, true, true, true,true); } @@ -235,7 +235,7 @@ public void getMailboxDataWithSignedSeqNr_ValidRemoveAddFromMessage() throws NoS this.testState.mockedStorage.getMailboxDataWithSignedSeqNr(mailboxStoragePayload, receiverKeys, receiverKeys.getPublic()); SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); - Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress(), true)); + Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, true, true, true, true,true); } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java index 472d65d7ea9..020ad2ff7ac 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java @@ -176,7 +176,7 @@ public void lostRemoveNeverUpdated() throws NoSuchAlgorithmException { // Seed node sees the remove, but client node does not seedNode.remove(getProtectedStorageEntry( - ownerKeys.getPublic(), protectedStoragePayload, 2), null, false); + ownerKeys.getPublic(), protectedStoragePayload, 2), null); GetDataRequest getDataRequest = clientNode.buildPreliminaryGetDataRequest(1); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java index cd1c27be1b6..9022ca2f659 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java @@ -119,8 +119,7 @@ boolean doRemove(ProtectedStorageEntry entry) { return true; } else { - // XXX: All callers just pass in true, a future patch can remove the argument. - return testState.mockedStorage.remove(entry, TestState.getTestNodeAddress(), true); + return testState.mockedStorage.remove(entry, TestState.getTestNodeAddress()); } } @@ -690,8 +689,7 @@ boolean doRemove(ProtectedStorageEntry entry) { return true; } else { - // XXX: All external callers just pass in true, a future patch can remove the argument. - return testState.mockedStorage.remove(entry, TestState.getTestNodeAddress(), true); + return testState.mockedStorage.remove(entry, TestState.getTestNodeAddress()); } } From 9f6913456805adf7bb43bd688c5f2a8c592e7e89 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 17:19:10 -0800 Subject: [PATCH 33/51] [REFACTOR] Clean up ClientAPI for refreshTTL Remove isDataOwner from the client API. All users pass in true. All test users don't care. --- .../main/java/bisq/core/offer/OfferBookService.java | 2 +- p2p/src/main/java/bisq/network/p2p/P2PService.java | 4 ++-- .../bisq/network/p2p/storage/P2PDataStorage.java | 12 ++++++++++++ .../p2p/storage/P2PDataStorageClientAPITest.java | 8 ++++---- .../P2PDataStorageProtectedStorageEntryTest.java | 3 +-- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/bisq/core/offer/OfferBookService.java b/core/src/main/java/bisq/core/offer/OfferBookService.java index a254f349e0b..6edea735e0a 100644 --- a/core/src/main/java/bisq/core/offer/OfferBookService.java +++ b/core/src/main/java/bisq/core/offer/OfferBookService.java @@ -164,7 +164,7 @@ public void refreshTTL(OfferPayload offerPayload, return; } - boolean result = p2PService.refreshTTL(offerPayload, true); + boolean result = p2PService.refreshTTL(offerPayload); if (result) { resultHandler.handleResult(); } else { diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index d4909f07c96..a78dc58d3d9 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -796,11 +796,11 @@ public boolean addProtectedStorageEntry(ProtectedStoragePayload protectedStorage } } - public boolean refreshTTL(ProtectedStoragePayload protectedStoragePayload, boolean isDataOwner) { + public boolean refreshTTL(ProtectedStoragePayload protectedStoragePayload) { if (isBootstrapped()) { try { RefreshOfferMessage refreshTTLMessage = p2PDataStorage.getRefreshTTLMessage(protectedStoragePayload, keyRing.getSignatureKeyPair()); - return p2PDataStorage.refreshTTL(refreshTTLMessage, networkNode.getNodeAddress(), isDataOwner); + return p2PDataStorage.refreshTTL(refreshTTLMessage, networkNode.getNodeAddress()); } catch (CryptoException e) { log.error("Signing at getDataWithSignedSeqNr failed. That should never happen."); return false; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index f267b11b108..706717ced66 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -648,7 +648,19 @@ private void broadcastProtectedStorageEntry(ProtectedStorageEntry protectedStora broadcast(new AddDataMessage(protectedStorageEntry), sender, broadcastListener, isDataOwner); } + /** + * Updates a local RefreshOffer with TTL changes and broadcasts those changes to the network + * + * @param refreshTTLMessage refreshTTLMessage containing the update + * @param sender local NodeAddress, if available + * @return true if the RefreshOffer was successfully updated and changes broadcast + */ public boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, + @Nullable NodeAddress sender) { + return refreshTTL(refreshTTLMessage, sender, true); + } + + private boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, @Nullable NodeAddress sender, boolean isDataOwner) { diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index 8ac4a409e61..340b579a1f7 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -123,7 +123,7 @@ public void getRefreshTTLMessage_NoExists() throws NoSuchAlgorithmException, Cry RefreshOfferMessage refreshOfferMessage = this.testState.mockedStorage.getRefreshTTLMessage(protectedStoragePayload, ownerKeys); SavedTestState beforeState = this.testState.saveTestState(refreshOfferMessage); - Assert.assertFalse(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress(), true)); + Assert.assertFalse(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress())); this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, false, true); } @@ -138,14 +138,14 @@ public void getRefreshTTLMessage() throws NoSuchAlgorithmException, CryptoExcept this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); RefreshOfferMessage refreshOfferMessage = this.testState.mockedStorage.getRefreshTTLMessage(protectedStoragePayload, ownerKeys); - this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress(), true); + this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress()); refreshOfferMessage = this.testState.mockedStorage.getRefreshTTLMessage(protectedStoragePayload, ownerKeys); this.testState.incrementClock(); SavedTestState beforeState = this.testState.saveTestState(refreshOfferMessage); - Assert.assertTrue(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress(), true)); + Assert.assertTrue(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress())); this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, true, true); } @@ -169,7 +169,7 @@ public void getRefreshTTLMessage_FirstOnMessageSecondAPI() throws NoSuchAlgorith this.testState.incrementClock(); SavedTestState beforeState = this.testState.saveTestState(refreshOfferMessage); - Assert.assertTrue(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress(), true)); + Assert.assertTrue(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress())); this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, true, true); } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java index 9022ca2f659..817e74b968e 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java @@ -146,8 +146,7 @@ boolean doRefreshTTL(RefreshOfferMessage refreshOfferMessage) { return true; } else { - // XXX: All external callers just pass in true for isDataOwner a future patch can remove the argument. - return this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress(), true); + return this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress()); } } From bfdb8f5715ceaa28feabd195a9e370173065fc5f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 16:54:53 -0800 Subject: [PATCH 34/51] Make isDataOwner a private policy decision in BroadcastHandler isDataOwner is used when deciding how many peer nodes should receive a BroadcastMessage. If the BroadcastMessage originated on the local node it is sent to ALL peer nodes with a small delay. If the node is only relaying the message (it originated on a different node) it is sent to MAX(peers.size(), 7) peers with a delay that is twice as long. All the information needed to determine whether or not the BroadcastMessage originated on the local node is available at the final broadcast site and there is no reason to have callers pass it in. In the event that the sender address is not known during broadcast (which is only a remote possibility due to how early the local node address is set during startup) we can default to relay mode. This first patch just removes the deep parameters. The next will remove everything else. There is one real change in LiteNodeNetworkService.java where it was using the local node when it should have been using the peer node. This was updated to the correct behavior. --- .../network/StateNetworkService.java | 2 +- .../full/network/FullNodeNetworkService.java | 2 +- .../lite/network/LiteNodeNetworkService.java | 2 +- .../network/p2p/peers/BroadcastHandler.java | 4 +++- .../bisq/network/p2p/peers/Broadcaster.java | 4 ++-- .../network/p2p/storage/P2PDataStorage.java | 4 ++-- .../P2PDataStorageOnMessageHandlerTest.java | 6 +++--- .../bisq/network/p2p/storage/TestState.java | 18 +++++++++--------- 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java b/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java index 1a0c4cc34af..69d6c7ea639 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java @@ -156,7 +156,7 @@ public void requestHashesFromAllConnectedSeedNodes(int fromHeight) { public void broadcastMyStateHash(StH myStateHash) { NewStateHashMessage newStateHashMessage = getNewStateHashMessage(myStateHash); - broadcaster.broadcast(newStateHashMessage, networkNode.getNodeAddress(), null, true); + broadcaster.broadcast(newStateHashMessage, networkNode.getNodeAddress(), null); } public void requestHashes(int fromHeight, String peersAddress) { diff --git a/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java b/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java index 97e8d5d56e7..bff48afb1ed 100644 --- a/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java +++ b/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java @@ -105,7 +105,7 @@ public void publishNewBlock(Block block) { log.info("Publish new block at height={} and block hash={}", block.getHeight(), block.getHash()); RawBlock rawBlock = RawBlock.fromBlock(block); NewBlockBroadcastMessage newBlockBroadcastMessage = new NewBlockBroadcastMessage(rawBlock); - broadcaster.broadcast(newBlockBroadcastMessage, networkNode.getNodeAddress(), null, true); + broadcaster.broadcast(newBlockBroadcastMessage, networkNode.getNodeAddress(), null); } diff --git a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java index 2cb081834f6..7fbab2f6b58 100644 --- a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java +++ b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java @@ -238,7 +238,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { log.debug("We received a new message from peer {} and broadcast it to our peers. extBlockId={}", connection.getPeersNodeAddressOptional().orElse(null), extBlockId); receivedBlocks.add(extBlockId); - broadcaster.broadcast(newBlockBroadcastMessage, networkNode.getNodeAddress(), null, false); + broadcaster.broadcast(newBlockBroadcastMessage, connection.getPeersNodeAddressOptional().orElse(null), null); listeners.forEach(listener -> listener.onNewBlockReceived(newBlockBroadcastMessage)); } else { log.debug("We had that message already and do not further broadcast it. extBlockId={}", extBlockId); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/BroadcastHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/BroadcastHandler.java index dde4062d20a..cc8469d8bda 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/BroadcastHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/BroadcastHandler.java @@ -111,7 +111,7 @@ public void cancel() { /////////////////////////////////////////////////////////////////////////////////////////// public void broadcast(BroadcastMessage message, @Nullable NodeAddress sender, ResultHandler resultHandler, - @Nullable Listener listener, boolean isDataOwner) { + @Nullable Listener listener) { this.message = message; this.resultHandler = resultHandler; this.listener = listener; @@ -127,6 +127,8 @@ public void broadcast(BroadcastMessage message, @Nullable NodeAddress sender, Re Collections.shuffle(connectedPeersList); numPeers = connectedPeersList.size(); int delay = 50; + + boolean isDataOwner = (sender != null) && sender.equals(networkNode.getNodeAddress()); if (!isDataOwner) { // for not data owner (relay nodes) we send to max. 7 nodes and use a longer delay numPeers = Math.min(7, connectedPeersList.size()); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/Broadcaster.java b/p2p/src/main/java/bisq/network/p2p/peers/Broadcaster.java index a5f85879e75..727c488b0e3 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/Broadcaster.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/Broadcaster.java @@ -56,9 +56,9 @@ public void shutDown() { /////////////////////////////////////////////////////////////////////////////////////////// public void broadcast(BroadcastMessage message, @Nullable NodeAddress sender, - @Nullable BroadcastHandler.Listener listener, boolean isDataOwner) { + @Nullable BroadcastHandler.Listener listener) { BroadcastHandler broadcastHandler = new BroadcastHandler(networkNode, peerManager); - broadcastHandler.broadcast(message, sender, this, listener, isDataOwner); + broadcastHandler.broadcast(message, sender, this, listener); broadcastHandlers.add(broadcastHandler); } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 706717ced66..76482f1cd4e 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -549,7 +549,7 @@ private boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, // Broadcast the payload if requested by caller if (allowBroadcast) - broadcaster.broadcast(new AddPersistableNetworkPayloadMessage(payload), sender, null, isDataOwner); + broadcaster.broadcast(new AddPersistableNetworkPayloadMessage(payload), sender, null); return true; } @@ -926,7 +926,7 @@ private boolean hasSequenceNrIncreased(int newSequenceNumber, ByteArray hashOfDa private void broadcast(BroadcastMessage message, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener listener, boolean isDataOwner) { - broadcaster.broadcast(message, sender, listener, isDataOwner); + broadcaster.broadcast(message, sender, listener); } public static ByteArray get32ByteHashAsByteArray(NetworkPayload data) { diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageOnMessageHandlerTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageOnMessageHandlerTest.java index 5a73e8acdd2..b5fe473732e 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageOnMessageHandlerTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageOnMessageHandlerTest.java @@ -69,7 +69,7 @@ public void invalidBroadcastMessage() { this.testState.mockedStorage.onMessage(envelope, mockedConnection); verify(this.testState.appendOnlyDataStoreListener, never()).onAdded(any(PersistableNetworkPayload.class)); - verify(this.testState.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), eq(null), anyBoolean()); + verify(this.testState.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), eq(null)); } @Test @@ -82,7 +82,7 @@ public void unsupportedBroadcastMessage() { this.testState.mockedStorage.onMessage(envelope, mockedConnection); verify(this.testState.appendOnlyDataStoreListener, never()).onAdded(any(PersistableNetworkPayload.class)); - verify(this.testState.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), eq(null), anyBoolean()); + verify(this.testState.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), eq(null)); } @Test @@ -96,6 +96,6 @@ public void invalidConnectionObject() { this.testState.mockedStorage.onMessage(envelope, mockedConnection); verify(this.testState.appendOnlyDataStoreListener, never()).onAdded(any(PersistableNetworkPayload.class)); - verify(this.testState.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), eq(null), anyBoolean()); + verify(this.testState.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), eq(null)); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index 51d831df41a..1e0a2aa2c8b 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -189,9 +189,9 @@ void verifyPersistableAdd(SavedTestState beforeState, if (expectedBroadcast) verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), any(NodeAddress.class), - eq(null), eq(expectedIsDataOwner)); + eq(null)); else - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); } void verifyProtectedStorageAdd(SavedTestState beforeState, @@ -222,13 +222,13 @@ void verifyProtectedStorageAdd(SavedTestState beforeState, if (expectedBroadcast) { final ArgumentCaptor captor = ArgumentCaptor.forClass(BroadcastMessage.class); verify(this.mockBroadcaster).broadcast(captor.capture(), any(NodeAddress.class), - eq(null), eq(expectedIsDataOwner)); + eq(null)); BroadcastMessage broadcastMessage = captor.getValue(); Assert.assertTrue(broadcastMessage instanceof AddDataMessage); Assert.assertEquals(protectedStorageEntry, ((AddDataMessage) broadcastMessage).getProtectedStorageEntry()); } else { - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); } if (expectedSequenceNrMapWrite) { @@ -280,7 +280,7 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, verify(this.mockSeqNrStorage, never()).queueUpForSave(any(SequenceNumberMap.class), anyLong()); if (!expectedBroadcast) - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); protectedStorageEntries.forEach(protectedStorageEntry -> { @@ -292,9 +292,9 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, if (expectedBroadcast) { if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) - verify(this.mockBroadcaster).broadcast(any(RemoveMailboxDataMessage.class), any(NodeAddress.class), eq(null), eq(expectedIsDataOwner)); + verify(this.mockBroadcaster).broadcast(any(RemoveMailboxDataMessage.class), any(NodeAddress.class), eq(null)); else - verify(this.mockBroadcaster).broadcast(any(RemoveDataMessage.class), any(NodeAddress.class), eq(null), eq(expectedIsDataOwner)); + verify(this.mockBroadcaster).broadcast(any(RemoveDataMessage.class), any(NodeAddress.class), eq(null)); } @@ -326,7 +326,7 @@ void verifyRefreshTTL(SavedTestState beforeState, final ArgumentCaptor captor = ArgumentCaptor.forClass(BroadcastMessage.class); verify(this.mockBroadcaster).broadcast(captor.capture(), any(NodeAddress.class), - eq(null), eq(expectedIsDataOwner)); + eq(null)); BroadcastMessage broadcastMessage = captor.getValue(); Assert.assertTrue(broadcastMessage instanceof RefreshOfferMessage); @@ -343,7 +343,7 @@ void verifyRefreshTTL(SavedTestState beforeState, Assert.assertEquals(beforeState.creationTimestampBeforeUpdate, entryAfterRefresh.getCreationTimeStamp()); } - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class), anyBoolean()); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); verify(this.mockSeqNrStorage, never()).queueUpForSave(any(SequenceNumberMap.class), anyLong()); } } From 4dc4532c91e979a21258bf5a67f194ec75126193 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 17:10:48 -0800 Subject: [PATCH 35/51] Remove isDataOwner from P2PDataStorage Remove all usages in code and tests now that the behavior is internal to the BroadcastHandler --- .../network/p2p/storage/P2PDataStorage.java | 48 +++++++------------ .../storage/P2PDataStorageClientAPITest.java | 18 +++---- .../P2PDataStorageGetDataIntegrationTest.java | 8 ++-- ...aStoragePersistableNetworkPayloadTest.java | 6 +-- .../P2PDataStorageProcessGetDataResponse.java | 26 +++++----- ...PDataStorageProtectedStorageEntryTest.java | 14 ++---- .../P2PDataStorageRemoveExpiredTest.java | 12 ++--- .../storage/P2PDataStoreDisconnectTest.java | 2 +- .../bisq/network/p2p/storage/TestState.java | 17 +++---- 9 files changed, 60 insertions(+), 91 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 76482f1cd4e..42faa491aa6 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -330,7 +330,7 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress long ts2 = System.currentTimeMillis(); dataSet.forEach(e -> { // We don't broadcast here (last param) as we are only connected to the seed node and would be pointless - addProtectedStorageEntry(e, sender, null, false, false); + addProtectedStorageEntry(e, sender, null, false); }); log.info("Processing {} protectedStorageEntries took {} ms.", dataSet.size(), this.clock.millis() - ts2); @@ -351,8 +351,7 @@ public void processGetDataResponse(GetDataResponse getDataResponse, NodeAddress } } else { // We don't broadcast here as we are only connected to the seed node and would be pointless - addPersistableNetworkPayload(e, sender, false, - false, false, false); + addPersistableNetworkPayload(e, sender,false, false, false); } }); log.info("Processing {} persistableNetworkPayloads took {} ms.", @@ -415,16 +414,16 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof BroadcastMessage) { connection.getPeersNodeAddressOptional().ifPresent(peersNodeAddress -> { if (networkEnvelope instanceof AddDataMessage) { - addProtectedStorageEntry(((AddDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, null, false, true); + addProtectedStorageEntry(((AddDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, null, true); } else if (networkEnvelope instanceof RemoveDataMessage) { - remove(((RemoveDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress, false); + remove(((RemoveDataMessage) networkEnvelope).getProtectedStorageEntry(), peersNodeAddress); } else if (networkEnvelope instanceof RemoveMailboxDataMessage) { - remove(((RemoveMailboxDataMessage) networkEnvelope).getProtectedMailboxStorageEntry(), peersNodeAddress, false); + remove(((RemoveMailboxDataMessage) networkEnvelope).getProtectedMailboxStorageEntry(), peersNodeAddress); } else if (networkEnvelope instanceof RefreshOfferMessage) { - refreshTTL((RefreshOfferMessage) networkEnvelope, peersNodeAddress, false); + refreshTTL((RefreshOfferMessage) networkEnvelope, peersNodeAddress); } else if (networkEnvelope instanceof AddPersistableNetworkPayloadMessage) { addPersistableNetworkPayload(((AddPersistableNetworkPayloadMessage) networkEnvelope).getPersistableNetworkPayload(), - peersNodeAddress, false, true, false, true); + peersNodeAddress, true, false, true); } }); } @@ -507,12 +506,11 @@ public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, @Nullable NodeAddress sender, boolean allowReBroadcast) { return addPersistableNetworkPayload( - payload, sender, true, true, allowReBroadcast, false); + payload, sender, true, allowReBroadcast, false); } private boolean addPersistableNetworkPayload(PersistableNetworkPayload payload, @Nullable NodeAddress sender, - boolean isDataOwner, boolean allowBroadcast, boolean reBroadcast, boolean checkDate) { @@ -580,13 +578,12 @@ private void addPersistableNetworkPayloadFromInitialRequest(PersistableNetworkPa */ public boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener listener) { - return addProtectedStorageEntry(protectedStorageEntry, sender, listener, true, true); + return addProtectedStorageEntry(protectedStorageEntry, sender, listener, true); } private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener listener, - boolean isDataOwner, boolean allowBroadcast) { ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); @@ -632,7 +629,7 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE // Optionally, broadcast the add/update depending on the calling environment if (allowBroadcast) - broadcastProtectedStorageEntry(protectedStorageEntry, sender, listener, isDataOwner); + broadcastProtectedStorageEntry(protectedStorageEntry, sender, listener); // Persist ProtectedStorageEntrys carrying PersistablePayload payloads if (protectedStoragePayload instanceof PersistablePayload) @@ -643,9 +640,8 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE private void broadcastProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, - @Nullable BroadcastHandler.Listener broadcastListener, - boolean isDataOwner) { - broadcast(new AddDataMessage(protectedStorageEntry), sender, broadcastListener, isDataOwner); + @Nullable BroadcastHandler.Listener broadcastListener) { + broadcast(new AddDataMessage(protectedStorageEntry), sender, broadcastListener); } /** @@ -657,12 +653,6 @@ private void broadcastProtectedStorageEntry(ProtectedStorageEntry protectedStora */ public boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, @Nullable NodeAddress sender) { - return refreshTTL(refreshTTLMessage, sender, true); - } - - private boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, - @Nullable NodeAddress sender, - boolean isDataOwner) { ByteArray hashOfPayload = new ByteArray(refreshTTLMessage.getHashOfPayload()); ProtectedStorageEntry storedData = map.get(hashOfPayload); @@ -698,7 +688,7 @@ private boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, sequenceNumberMapStorage.queueUpForSave(SequenceNumberMap.clone(sequenceNumberMap), 1000); // Always broadcast refreshes - broadcast(refreshTTLMessage, sender, null, isDataOwner); + broadcast(refreshTTLMessage, sender, null); return true; } @@ -713,12 +703,6 @@ private boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, */ public boolean remove(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender) { - return remove(protectedStorageEntry, sender, true); - } - - private boolean remove(ProtectedStorageEntry protectedStorageEntry, - @Nullable NodeAddress sender, - boolean isDataOwner) { ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); @@ -752,9 +736,9 @@ private boolean remove(ProtectedStorageEntry protectedStorageEntry, printData("after remove"); if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) { - broadcast(new RemoveMailboxDataMessage((ProtectedMailboxStorageEntry) protectedStorageEntry), sender, null, isDataOwner); + broadcast(new RemoveMailboxDataMessage((ProtectedMailboxStorageEntry) protectedStorageEntry), sender, null); } else { - broadcast(new RemoveDataMessage(protectedStorageEntry), sender, null, isDataOwner); + broadcast(new RemoveDataMessage(protectedStorageEntry), sender, null); } return true; @@ -925,7 +909,7 @@ private boolean hasSequenceNrIncreased(int newSequenceNumber, ByteArray hashOfDa } private void broadcast(BroadcastMessage message, @Nullable NodeAddress sender, - @Nullable BroadcastHandler.Listener listener, boolean isDataOwner) { + @Nullable BroadcastHandler.Listener listener) { broadcaster.broadcast(message, sender, listener); } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index 340b579a1f7..5c5dad4ea75 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -73,7 +73,7 @@ public void getProtectedStorageEntry_NoExist() throws NoSuchAlgorithmException, SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true); } // TESTCASE: Adding an entry from the getProtectedStorageEntry API of an existing item correctly updates the item @@ -90,7 +90,7 @@ public void getProtectedStorageEntry() throws NoSuchAlgorithmException, CryptoEx protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true); } // TESTCASE: Adding an entry from the getProtectedStorageEntry API of an existing item (added from onMessage path) correctly updates the item @@ -110,7 +110,7 @@ public void getProtectedStorageEntry_FirstOnMessageSecondAPI() throws NoSuchAlgo protectedStorageEntry = this.testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); Assert.assertTrue(this.testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null)); - this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true, true); + this.testState.verifyProtectedStorageAdd(beforeState, protectedStorageEntry, true, true, true, true); } // TESTCASE: Updating an entry from the getRefreshTTLMessage API correctly errors if the item hasn't been seen @@ -125,7 +125,7 @@ public void getRefreshTTLMessage_NoExists() throws NoSuchAlgorithmException, Cry SavedTestState beforeState = this.testState.saveTestState(refreshOfferMessage); Assert.assertFalse(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress())); - this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, false, true); + this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, false); } // TESTCASE: Updating an entry from the getRefreshTTLMessage API correctly "refreshes" the item @@ -147,7 +147,7 @@ public void getRefreshTTLMessage() throws NoSuchAlgorithmException, CryptoExcept SavedTestState beforeState = this.testState.saveTestState(refreshOfferMessage); Assert.assertTrue(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress())); - this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, true, true); + this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, true); } // TESTCASE: Updating an entry from the getRefreshTTLMessage API correctly "refreshes" the item when it was originally added from onMessage path @@ -171,7 +171,7 @@ public void getRefreshTTLMessage_FirstOnMessageSecondAPI() throws NoSuchAlgorith SavedTestState beforeState = this.testState.saveTestState(refreshOfferMessage); Assert.assertTrue(this.testState.mockedStorage.refreshTTL(refreshOfferMessage, TestState.getTestNodeAddress())); - this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, true, true); + this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, true); } // TESTCASE: Removing a non-existent mailbox entry from the getMailboxDataWithSignedSeqNr API @@ -188,7 +188,7 @@ public void getMailboxDataWithSignedSeqNr_RemoveNoExist() throws NoSuchAlgorithm SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); - this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, false, false, false, true, true); + this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, false, false, false, true); } // TESTCASE: Adding, then removing a mailbox message from the getMailboxDataWithSignedSeqNr API @@ -210,7 +210,7 @@ public void getMailboxDataWithSignedSeqNr_AddThenRemove() throws NoSuchAlgorithm SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); - this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, true, true, true, true,true); + this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, true, true, true, true); } // TESTCASE: Removing a mailbox message that was added from the onMessage handler @@ -237,6 +237,6 @@ public void getMailboxDataWithSignedSeqNr_ValidRemoveAddFromMessage() throws NoS SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); - this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, true, true, true, true,true); + this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, true, true, true, true); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java index 020ad2ff7ac..21e48d2757b 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageGetDataIntegrationTest.java @@ -88,7 +88,7 @@ public void basicSynchronizationWorks() throws NoSuchAlgorithmException { clientNode.processGetDataResponse(getDataResponse, null); clientNodeTestState.verifyProtectedStorageAdd( - beforeState, onSeedNode, true, true, false, true, false); + beforeState, onSeedNode, true, true, false, true); } // TESTCASE: Synchronization after peer restart works for in-memory ProtectedStorageEntrys @@ -118,7 +118,7 @@ public void basicSynchronizationWorksAfterRestartTransient() throws NoSuchAlgori clientNode.processGetDataResponse(getDataResponse, null); clientNodeTestState.verifyProtectedStorageAdd( - beforeState, transientEntry, true, true, false, true, false); + beforeState, transientEntry, true, true, false, true); } // TESTCASE: Synchronization after peer restart works for in-memory ProtectedStorageEntrys @@ -151,7 +151,7 @@ public void basicSynchronizationWorksAfterRestartPersistent() throws NoSuchAlgor clientNode.processGetDataResponse(getDataResponse, null); clientNodeTestState.verifyProtectedStorageAdd( - beforeState, persistentEntry, false, false, false, false, false); + beforeState, persistentEntry, false, false, false, false); Assert.assertTrue(clientNodeTestState.mockedStorage.getMap().containsValue(persistentEntry)); } @@ -188,6 +188,6 @@ public void lostRemoveNeverUpdated() throws NoSuchAlgorithmException { // Should succeed clientNodeTestState.verifyProtectedStorageRemove( - beforeState, onSeedNodeAndClientNode, false, false, false, false, false); + beforeState, onSeedNodeAndClientNode, false, false, false, false); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java index d0042df1f2f..89cac4afcb3 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java @@ -76,10 +76,6 @@ enum TestCase { ON_MESSAGE, } - boolean expectedIsDataOwner() { - return this.testCase == TestCase.PUBLIC_API; - } - void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean expectedReturnValue, boolean expectedStateChange) { SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); @@ -93,7 +89,7 @@ void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean testState.mockedStorage.onMessage(new AddPersistableNetworkPayloadMessage(persistableNetworkPayload), mockedConnection); } - this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedStateChange, expectedStateChange, this.expectedIsDataOwner()); + this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedStateChange, expectedStateChange); } @Before diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java index abf605ae1a6..0a3c7cf9184 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProcessGetDataResponse.java @@ -117,7 +117,7 @@ public void processGetDataResponse_newPNPUpdatesState() { TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, persistableNetworkPayload, true, true, false, false); + beforeState, persistableNetworkPayload, true, true, false); } // TESTCASE: GetDataResponse w/ invalid PNP does nothing (LazyProcessed) @@ -130,7 +130,7 @@ public void processGetDataResponse_newInvalidPNPDoesNothing() { TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, persistableNetworkPayload, false, false, false, false); + beforeState, persistableNetworkPayload, false, false, false); } // TESTCASE: GetDataResponse w/ existing PNP changes no state @@ -145,7 +145,7 @@ public void processGetDataResponse_duplicatePNPDoesNothing() { TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, persistableNetworkPayload, false, false, false, false); + beforeState, persistableNetworkPayload, false, false, false); } // TESTCASE: GetDataResponse w/ missing PNP is added with no broadcast or listener signal (ProcessOncePersistableNetworkPayload) @@ -158,7 +158,7 @@ public void processGetDataResponse_newPNPUpdatesState_LazyProcessed() { TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, persistableNetworkPayload, true, false, false, false); + beforeState, persistableNetworkPayload, true, false, false); } // TESTCASE: GetDataResponse w/ existing PNP changes no state (ProcessOncePersistableNetworkPayload) @@ -173,7 +173,7 @@ public void processGetDataResponse_duplicatePNPDoesNothing_LazyProcessed() { TestState.SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, persistableNetworkPayload, false, false, false, false); + beforeState, persistableNetworkPayload, false, false, false); } // TESTCASE: Second call to processGetDataResponse adds PNP for non-ProcessOncePersistableNetworkPayloads @@ -185,14 +185,14 @@ public void processGetDataResponse_secondProcessNewPNPUpdatesState() { TestState.SavedTestState beforeState = this.testState.saveTestState(addFromFirstProcess); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, addFromFirstProcess, true, true, false, false); + beforeState, addFromFirstProcess, true, true, false); PersistableNetworkPayload addFromSecondProcess = new PersistableNetworkPayloadStub(new byte[] { 2 }); getDataResponse = buildGetDataResponse(addFromSecondProcess); beforeState = this.testState.saveTestState(addFromSecondProcess); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, addFromSecondProcess, true, true, false, false); + beforeState, addFromSecondProcess, true, true, false); } // TESTCASE: Second call to processGetDataResponse does not add any PNP (LazyProcessed) @@ -204,14 +204,14 @@ public void processGetDataResponse_secondProcessNoPNPUpdates_LazyProcessed() { TestState.SavedTestState beforeState = this.testState.saveTestState(addFromFirstProcess); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, addFromFirstProcess, true, false, false, false); + beforeState, addFromFirstProcess, true, false, false); PersistableNetworkPayload addFromSecondProcess = new LazyPersistableNetworkPayloadStub(new byte[] { 2 }); getDataResponse = buildGetDataResponse(addFromSecondProcess); beforeState = this.testState.saveTestState(addFromSecondProcess); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyPersistableAdd( - beforeState, addFromSecondProcess, false, false, false, false); + beforeState, addFromSecondProcess, false, false, false); } // TESTCASE: GetDataResponse w/ missing PSE is added with no broadcast or listener signal @@ -224,7 +224,7 @@ public void processGetDataResponse_newPSEUpdatesState() throws NoSuchAlgorithmEx TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyProtectedStorageAdd( - beforeState, protectedStorageEntry, true, true, false, true, false); + beforeState, protectedStorageEntry, true, true, false, true); } // TESTCASE: GetDataResponse w/ existing PSE changes no state @@ -238,7 +238,7 @@ public void processGetDataResponse_duplicatePSEDoesNothing() throws NoSuchAlgori this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.verifyProtectedStorageAdd( - beforeState, protectedStorageEntry, false, false, false, false, false); + beforeState, protectedStorageEntry, false, false, false, false); } // TESTCASE: GetDataResponse w/ missing PSE is added with no broadcast or listener signal @@ -251,13 +251,13 @@ public void processGetDataResponse_secondCallNewPSEUpdatesState() throws NoSuchA TestState.SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyProtectedStorageAdd( - beforeState, protectedStorageEntry, true, true, false, true, false); + beforeState, protectedStorageEntry, true, true, false, true); protectedStorageEntry = getProtectedStorageEntryForAdd(); getDataResponse = buildGetDataResponse(protectedStorageEntry); beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.processGetDataResponse(getDataResponse, this.peerNodeAddress); this.testState.verifyProtectedStorageAdd( - beforeState, protectedStorageEntry, true, true, false, true, false); + beforeState, protectedStorageEntry, true, true, false, true); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java index 817e74b968e..a6d34d93d19 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageProtectedStorageEntryTest.java @@ -83,12 +83,6 @@ abstract public static class ProtectedStorageEntryTestBase { @Parameterized.Parameter(0) public boolean useMessageHandler; - boolean expectIsDataOwner() { - // The onMessage handler variant should always broadcast with isDataOwner == false - // The Client API should always broadcast with isDataOwner == true - return !useMessageHandler; - } - @Parameterized.Parameters(name = "{index}: Test with useMessageHandler={0}") public static Collection data() { List data = new ArrayList<>(); @@ -195,10 +189,10 @@ void doProtectedStorageAddAndVerify(ProtectedStorageEntry protectedStorageEntry, if (expectedStateChange) { this.testState.verifyProtectedStorageAdd( - beforeState, protectedStorageEntry, true, true, true, true, this.expectIsDataOwner()); + beforeState, protectedStorageEntry, true, true, true, true); } else{ this.testState.verifyProtectedStorageAdd( - beforeState, protectedStorageEntry, false, false, false, false, this.expectIsDataOwner()); + beforeState, protectedStorageEntry, false, false, false, false); } } @@ -216,7 +210,7 @@ void doProtectedStorageRemoveAndVerify(ProtectedStorageEntry entry, if (!this.useMessageHandler) Assert.assertEquals(expectedReturnValue, addResult); - this.testState.verifyProtectedStorageRemove(beforeState, entry, expectedHashMapAndDataStoreUpdated, expectedListenersSignaled, expectedBroadcast, expectedSeqNrWrite, this.expectIsDataOwner()); + this.testState.verifyProtectedStorageRemove(beforeState, entry, expectedHashMapAndDataStoreUpdated, expectedListenersSignaled, expectedBroadcast, expectedSeqNrWrite); } /// Valid Add Tests (isValidForAdd() and matchesRelevantPubKey() return true) @@ -492,7 +486,7 @@ void doRefreshTTLAndVerify(RefreshOfferMessage refreshOfferMessage, boolean expe if (!this.useMessageHandler) Assert.assertEquals(expectedReturnValue, returnValue); - this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, expectStateChange, this.expectIsDataOwner()); + this.testState.verifyRefreshTTL(beforeState, refreshOfferMessage, expectStateChange); } // TESTCASE: Refresh an entry that doesn't exist diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java index f561733e375..f6b8c659109 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageRemoveExpiredTest.java @@ -67,7 +67,7 @@ public void removeExpiredEntries_SkipsNonExpirableEntries() throws NoSuchAlgorit SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); - this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, false, false, false, false, false); + this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, false, false, false, false); } // TESTCASE: Correctly skips all PersistableNetworkPayloads since they are not expirable @@ -93,7 +93,7 @@ public void removeExpiredEntries_SkipNonExpiredExpirableEntries() throws CryptoE SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); - this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, false, false, false, false, false); + this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, false, false, false, false); } // TESTCASE: Correctly expires non-persistable entries that are expired @@ -110,7 +110,7 @@ public void removeExpiredEntries_ExpiresExpiredExpirableEntries() throws CryptoE SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); - this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, true, true, false, false, false); + this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, true, true, false, false); } // TESTCASE: Correctly skips persistable entries that are not expired @@ -124,7 +124,7 @@ public void removeExpiredEntries_SkipNonExpiredPersistableExpirableEntries() thr SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); - this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, false, false, false, false, false); + this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, false, false, false, false); } // TESTCASE: Correctly expires persistable entries that are expired @@ -141,7 +141,7 @@ public void removeExpiredEntries_ExpiresExpiredPersistableExpirableEntries() thr SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); - this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, true, true, false, false, false); + this.testState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, true, true, false, false); } // TESTCASE: Ensure we try to purge old entries sequence number map when size exceeds the maximum size @@ -187,6 +187,6 @@ public void removeExpiredEntries_PurgeSeqNrMap() throws CryptoException, NoSuchA // The first 4 entries (11 days old) should be purged from the SequenceNumberMap SavedTestState beforeState = this.testState.saveTestState(purgedProtectedStorageEntry); this.testState.mockedStorage.removeExpiredEntries(); - this.testState.verifyProtectedStorageRemove(beforeState, expectedRemoves, true, true, false, false, false); + this.testState.verifyProtectedStorageRemove(beforeState, expectedRemoves, true, true, false, false); } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java index abba32569c8..408edac93f8 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java @@ -69,7 +69,7 @@ private static void verifyStateAfterDisconnect(TestState currentState, ProtectedStorageEntry protectedStorageEntry = beforeState.protectedStorageEntryBeforeOp; currentState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, - wasRemoved, wasRemoved, false, false, false); + wasRemoved, wasRemoved, false, false); if (wasTTLReduced) Assert.assertTrue(protectedStorageEntry.getCreationTimeStamp() < beforeState.creationTimestampBeforeUpdate); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index 1e0a2aa2c8b..4a9a907b78f 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -173,8 +173,7 @@ void verifyPersistableAdd(SavedTestState beforeState, PersistableNetworkPayload persistableNetworkPayload, boolean expectedHashMapAndDataStoreUpdated, boolean expectedListenersSignaled, - boolean expectedBroadcast, - boolean expectedIsDataOwner) { + boolean expectedBroadcast) { P2PDataStorage.ByteArray hash = new P2PDataStorage.ByteArray(persistableNetworkPayload.getHash()); if (expectedHashMapAndDataStoreUpdated) @@ -199,8 +198,7 @@ void verifyProtectedStorageAdd(SavedTestState beforeState, boolean expectedHashMapAndDataStoreUpdated, boolean expectedListenersSignaled, boolean expectedBroadcast, - boolean expectedSequenceNrMapWrite, - boolean expectedIsDataOwner) { + boolean expectedSequenceNrMapWrite) { P2PDataStorage.ByteArray hashMapHash = P2PDataStorage.get32ByteHashAsByteArray(protectedStorageEntry.getProtectedStoragePayload()); if (expectedHashMapAndDataStoreUpdated) { @@ -243,12 +241,11 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, boolean expectedHashMapAndDataStoreUpdated, boolean expectedListenersSignaled, boolean expectedBroadcast, - boolean expectedSeqNrWrite, - boolean expectedIsDataOwner) { + boolean expectedSeqNrWrite) { verifyProtectedStorageRemove(beforeState, Collections.singletonList(protectedStorageEntry), expectedHashMapAndDataStoreUpdated, expectedListenersSignaled, expectedBroadcast, - expectedSeqNrWrite, expectedIsDataOwner); + expectedSeqNrWrite); } void verifyProtectedStorageRemove(SavedTestState beforeState, @@ -256,8 +253,7 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, boolean expectedHashMapAndDataStoreUpdated, boolean expectedListenersSignaled, boolean expectedBroadcast, - boolean expectedSeqNrWrite, - boolean expectedIsDataOwner) { + boolean expectedSeqNrWrite) { // The default matcher expects orders to stay the same. So, create a custom matcher function since // we don't care about the order. @@ -312,8 +308,7 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, void verifyRefreshTTL(SavedTestState beforeState, RefreshOfferMessage refreshOfferMessage, - boolean expectedStateChange, - boolean expectedIsDataOwner) { + boolean expectedStateChange) { P2PDataStorage.ByteArray payloadHash = new P2PDataStorage.ByteArray(refreshOfferMessage.getHashOfPayload()); ProtectedStorageEntry entryAfterRefresh = this.mockedStorage.getMap().get(payloadHash); From 6ff8756dd8d34487b04cf4d96741ae5f09f21619 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 19:01:50 -0800 Subject: [PATCH 36/51] [REFACTOR] inline broadcast() private function Now that it is identical the the broadcaster version it can be inlined. --- .../bisq/network/p2p/storage/P2PDataStorage.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 42faa491aa6..160130d48c4 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -641,7 +641,7 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE private void broadcastProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender, @Nullable BroadcastHandler.Listener broadcastListener) { - broadcast(new AddDataMessage(protectedStorageEntry), sender, broadcastListener); + broadcaster.broadcast(new AddDataMessage(protectedStorageEntry), sender, broadcastListener); } /** @@ -688,7 +688,7 @@ public boolean refreshTTL(RefreshOfferMessage refreshTTLMessage, sequenceNumberMapStorage.queueUpForSave(SequenceNumberMap.clone(sequenceNumberMap), 1000); // Always broadcast refreshes - broadcast(refreshTTLMessage, sender, null); + broadcaster.broadcast(refreshTTLMessage, sender, null); return true; } @@ -736,9 +736,9 @@ public boolean remove(ProtectedStorageEntry protectedStorageEntry, printData("after remove"); if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) { - broadcast(new RemoveMailboxDataMessage((ProtectedMailboxStorageEntry) protectedStorageEntry), sender, null); + broadcaster.broadcast(new RemoveMailboxDataMessage((ProtectedMailboxStorageEntry) protectedStorageEntry), sender, null); } else { - broadcast(new RemoveDataMessage(protectedStorageEntry), sender, null); + broadcaster.broadcast(new RemoveDataMessage(protectedStorageEntry), sender, null); } return true; @@ -908,11 +908,6 @@ private boolean hasSequenceNrIncreased(int newSequenceNumber, ByteArray hashOfDa } } - private void broadcast(BroadcastMessage message, @Nullable NodeAddress sender, - @Nullable BroadcastHandler.Listener listener) { - broadcaster.broadcast(message, sender, listener); - } - public static ByteArray get32ByteHashAsByteArray(NetworkPayload data) { return new ByteArray(P2PDataStorage.get32ByteHash(data)); } From 5a174d546a4f43a51c0688fa0a2bb0bc2f3b3d80 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 19:03:08 -0800 Subject: [PATCH 37/51] [REFACTOR] inline broadcastProtectedStorageEntry() private function Now that it is identical the the broadcaster version it can be inlined. --- .../java/bisq/network/p2p/storage/P2PDataStorage.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 160130d48c4..44dfec89fb3 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -629,7 +629,7 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE // Optionally, broadcast the add/update depending on the calling environment if (allowBroadcast) - broadcastProtectedStorageEntry(protectedStorageEntry, sender, listener); + broadcaster.broadcast(new AddDataMessage(protectedStorageEntry), sender, listener); // Persist ProtectedStorageEntrys carrying PersistablePayload payloads if (protectedStoragePayload instanceof PersistablePayload) @@ -637,13 +637,7 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE return true; } - - private void broadcastProtectedStorageEntry(ProtectedStorageEntry protectedStorageEntry, - @Nullable NodeAddress sender, - @Nullable BroadcastHandler.Listener broadcastListener) { - broadcaster.broadcast(new AddDataMessage(protectedStorageEntry), sender, broadcastListener); - } - + /** * Updates a local RefreshOffer with TTL changes and broadcasts those changes to the network * From 1bd450b3bc3ff4f6c19d7e8f85bd3407a1c3e0ca Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 21 Nov 2019 19:07:15 -0800 Subject: [PATCH 38/51] [REFACTOR] inline maybeAddToRemoveAddOncePayloads() private function Now that there is only one user it can be inlined. --- .../bisq/network/p2p/storage/P2PDataStorage.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 44dfec89fb3..9371541fa1c 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -637,7 +637,7 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE return true; } - + /** * Updates a local RefreshOffer with TTL changes and broadcasts those changes to the network * @@ -717,7 +717,9 @@ public boolean remove(ProtectedStorageEntry protectedStorageEntry, sequenceNumberMap.put(hashOfPayload, new MapValue(protectedStorageEntry.getSequenceNumber(), this.clock.millis())); sequenceNumberMapStorage.queueUpForSave(SequenceNumberMap.clone(sequenceNumberMap), 300); - maybeAddToRemoveAddOncePayloads(protectedStoragePayload, hashOfPayload); + // Update that we have seen this AddOncePayload so the next time it is seen it fails verification + if (protectedStoragePayload instanceof AddOncePayload) + removedAddOncePayloads.add(hashOfPayload); if (storedEntry != null) { // Valid remove entry, do the remove and signal listeners @@ -769,13 +771,6 @@ public void removeInvalidProtectedStorageEntry(ProtectedStorageEntry protectedSt // source (network). } - private void maybeAddToRemoveAddOncePayloads(ProtectedStoragePayload protectedStoragePayload, - ByteArray hashOfData) { - if (protectedStoragePayload instanceof AddOncePayload) { - removedAddOncePayloads.add(hashOfData); - } - } - public ProtectedStorageEntry getProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, KeyPair ownerStoragePubKey) throws CryptoException { From 17f4b7096e39b5c5272606df8063a1c8d9bb92b6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 22 Nov 2019 17:51:35 -0800 Subject: [PATCH 39/51] [TESTS] Clean up mockito never() and eq(null) usages never() and any() don't play well together for nullable types. Change the broadcast mocks to user nullable() and fixup tests. --- .../storage/P2PDataStorageClientAPITest.java | 2 +- ...aStoragePersistableNetworkPayloadTest.java | 21 ++++++++++-------- .../bisq/network/p2p/storage/TestState.java | 22 +++++++++---------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java index 5c5dad4ea75..789411af399 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStorageClientAPITest.java @@ -188,7 +188,7 @@ public void getMailboxDataWithSignedSeqNr_RemoveNoExist() throws NoSuchAlgorithm SavedTestState beforeState = this.testState.saveTestState(protectedMailboxStorageEntry); Assert.assertTrue(this.testState.mockedStorage.remove(protectedMailboxStorageEntry, TestState.getTestNodeAddress())); - this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, false, false, false, true); + this.testState.verifyProtectedStorageRemove(beforeState, protectedMailboxStorageEntry, false, false, true, true); } // TESTCASE: Adding, then removing a mailbox message from the getMailboxDataWithSignedSeqNr API diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java index 89cac4afcb3..8f1dafd8d72 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoragePersistableNetworkPayloadTest.java @@ -76,7 +76,11 @@ enum TestCase { ON_MESSAGE, } - void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean expectedReturnValue, boolean expectedStateChange) { + void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, + boolean expectedReturnValue, + boolean expectedHashMapAndDataStoreUpdated, + boolean expectedListenersSignaled, + boolean expectedBroadcast) { SavedTestState beforeState = this.testState.saveTestState(persistableNetworkPayload); if (this.testCase == TestCase.PUBLIC_API) { @@ -89,7 +93,7 @@ void doAddAndVerify(PersistableNetworkPayload persistableNetworkPayload, boolean testState.mockedStorage.onMessage(new AddPersistableNetworkPayloadMessage(persistableNetworkPayload), mockedConnection); } - this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedStateChange, expectedStateChange, expectedStateChange); + this.testState.verifyPersistableAdd(beforeState, persistableNetworkPayload, expectedHashMapAndDataStoreUpdated, expectedListenersSignaled, expectedBroadcast); } @Before @@ -119,16 +123,15 @@ public static Collection data() { @Test public void addPersistableNetworkPayload() { // First add should succeed regardless of parameters - doAddAndVerify(this.persistableNetworkPayload, true, true); + doAddAndVerify(this.persistableNetworkPayload, true, true, true, true); } @Test public void addPersistableNetworkPayloadDuplicate() { - doAddAndVerify(this.persistableNetworkPayload, true, true); + doAddAndVerify(this.persistableNetworkPayload, true, true, true, true); - // Second call only succeeds if reBroadcast was set - boolean expectedReturnValue = this.reBroadcast; - doAddAndVerify(this.persistableNetworkPayload, expectedReturnValue, false); + // We return true and broadcast if reBroadcast is set + doAddAndVerify(this.persistableNetworkPayload, this.reBroadcast, false, false, this.reBroadcast); } } @@ -145,7 +148,7 @@ PersistableNetworkPayloadStub createInstance() { public void invalidHash() { PersistableNetworkPayload persistableNetworkPayload = new PersistableNetworkPayloadStub(false); - doAddAndVerify(persistableNetworkPayload, false, false); + doAddAndVerify(persistableNetworkPayload, false, false, false, false); } } @@ -168,7 +171,7 @@ public void outOfTolerance() { // The onMessage path checks for tolerance boolean expectedReturn = this.testCase != TestCase.ON_MESSAGE; - doAddAndVerify(persistableNetworkPayload, expectedReturn, expectedReturn); + doAddAndVerify(persistableNetworkPayload, expectedReturn, expectedReturn, expectedReturn, expectedReturn); } } } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java index 4a9a907b78f..0ede4a03c8e 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/TestState.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/TestState.java @@ -187,10 +187,10 @@ void verifyPersistableAdd(SavedTestState beforeState, verify(this.appendOnlyDataStoreListener, never()).onAdded(persistableNetworkPayload); if (expectedBroadcast) - verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), any(NodeAddress.class), - eq(null)); + verify(this.mockBroadcaster).broadcast(any(AddPersistableNetworkPayloadMessage.class), + nullable(NodeAddress.class), isNull()); else - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), nullable(NodeAddress.class), nullable(BroadcastHandler.Listener.class)); } void verifyProtectedStorageAdd(SavedTestState beforeState, @@ -219,14 +219,13 @@ void verifyProtectedStorageAdd(SavedTestState beforeState, if (expectedBroadcast) { final ArgumentCaptor captor = ArgumentCaptor.forClass(BroadcastMessage.class); - verify(this.mockBroadcaster).broadcast(captor.capture(), any(NodeAddress.class), - eq(null)); + verify(this.mockBroadcaster).broadcast(captor.capture(), nullable(NodeAddress.class), isNull()); BroadcastMessage broadcastMessage = captor.getValue(); Assert.assertTrue(broadcastMessage instanceof AddDataMessage); Assert.assertEquals(protectedStorageEntry, ((AddDataMessage) broadcastMessage).getProtectedStorageEntry()); } else { - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), nullable(NodeAddress.class), nullable(BroadcastHandler.Listener.class)); } if (expectedSequenceNrMapWrite) { @@ -276,7 +275,7 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, verify(this.mockSeqNrStorage, never()).queueUpForSave(any(SequenceNumberMap.class), anyLong()); if (!expectedBroadcast) - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), nullable(NodeAddress.class), nullable(BroadcastHandler.Listener.class)); protectedStorageEntries.forEach(protectedStorageEntry -> { @@ -288,9 +287,9 @@ void verifyProtectedStorageRemove(SavedTestState beforeState, if (expectedBroadcast) { if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) - verify(this.mockBroadcaster).broadcast(any(RemoveMailboxDataMessage.class), any(NodeAddress.class), eq(null)); + verify(this.mockBroadcaster).broadcast(any(RemoveMailboxDataMessage.class), nullable(NodeAddress.class), isNull()); else - verify(this.mockBroadcaster).broadcast(any(RemoveDataMessage.class), any(NodeAddress.class), eq(null)); + verify(this.mockBroadcaster).broadcast(any(RemoveDataMessage.class), nullable(NodeAddress.class), isNull()); } @@ -320,8 +319,7 @@ void verifyRefreshTTL(SavedTestState beforeState, Assert.assertTrue(entryAfterRefresh.getCreationTimeStamp() > beforeState.creationTimestampBeforeUpdate); final ArgumentCaptor captor = ArgumentCaptor.forClass(BroadcastMessage.class); - verify(this.mockBroadcaster).broadcast(captor.capture(), any(NodeAddress.class), - eq(null)); + verify(this.mockBroadcaster).broadcast(captor.capture(), nullable(NodeAddress.class), isNull()); BroadcastMessage broadcastMessage = captor.getValue(); Assert.assertTrue(broadcastMessage instanceof RefreshOfferMessage); @@ -338,7 +336,7 @@ void verifyRefreshTTL(SavedTestState beforeState, Assert.assertEquals(beforeState.creationTimestampBeforeUpdate, entryAfterRefresh.getCreationTimeStamp()); } - verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), any(NodeAddress.class), any(BroadcastHandler.Listener.class)); + verify(this.mockBroadcaster, never()).broadcast(any(BroadcastMessage.class), nullable(NodeAddress.class), nullable(BroadcastHandler.Listener.class)); verify(this.mockSeqNrStorage, never()).queueUpForSave(any(SequenceNumberMap.class), anyLong()); } } From b6b00268af77040d3070a835dcda6c7fe202509d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 23 Nov 2019 13:46:19 -0800 Subject: [PATCH 40/51] Remove ProtectedStorageEntry::updateSignature The only users were tests that can just pass a bad signature directly into the constructor. --- .../storage/payload/ProtectedStorageEntry.java | 7 +------ .../ProtectedMailboxStorageEntryTest.java | 15 +++++++-------- .../payload/ProtectedStorageEntryTest.java | 16 ++++++++++------ 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java index 1982052bc2b..87782af57a3 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java @@ -47,7 +47,7 @@ public class ProtectedStorageEntry implements NetworkPayload, PersistablePayload private final byte[] ownerPubKeyBytes; transient private final PublicKey ownerPubKey; private final int sequenceNumber; - private byte[] signature; + private final byte[] signature; private long creationTimeStamp; public ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, @@ -146,11 +146,6 @@ public void backDate() { creationTimeStamp -= ((ExpirablePayload) protectedStoragePayload).getTTL() / 2; } - // TODO: only used in tests so find a better way to test and delete public API - public void updateSignature(byte[] signature) { - this.signature = signature; - } - public boolean isExpired(Clock clock) { return protectedStoragePayload instanceof ExpirablePayload && (clock.millis() - creationTimeStamp) > ((ExpirablePayload) protectedStoragePayload).getTTL(); diff --git a/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntryTest.java index 74d466aa148..035e9e03d57 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntryTest.java @@ -107,14 +107,13 @@ public void isValidForAddOperation_EntryReceiverPayloadReceiverMismatch() throws // TESTCASE: validForAddOperation() should fail if the signature isn't valid @Test - public void isValidForAddOperation_BadSignature() throws NoSuchAlgorithmException, CryptoException { + public void isValidForAddOperation_BadSignature() throws NoSuchAlgorithmException { KeyPair senderKeys = TestUtils.generateKeyPair(); KeyPair receiverKeys = TestUtils.generateKeyPair(); MailboxStoragePayload mailboxStoragePayload = buildMailboxStoragePayload(senderKeys.getPublic(), receiverKeys.getPublic()); - ProtectedStorageEntry protectedStorageEntry = buildProtectedMailboxStorageEntry(mailboxStoragePayload, senderKeys, receiverKeys.getPublic(), 1); - - protectedStorageEntry.updateSignature( new byte[] { 0 }); + ProtectedStorageEntry protectedStorageEntry = new ProtectedMailboxStorageEntry( + mailboxStoragePayload, senderKeys.getPublic(), 1, new byte[] { 0 }, receiverKeys.getPublic(), Clock.systemDefaultZone()); Assert.assertFalse(protectedStorageEntry.isValidForAddOperation()); } @@ -145,14 +144,14 @@ public void validForRemoveEntryOwnerPayloadOwnerMismatch() throws NoSuchAlgorith // TESTCASE: isValidForRemoveOperation() should fail if the signature is bad @Test - public void isValidForRemoveOperation_BadSignature() throws NoSuchAlgorithmException, CryptoException { + public void isValidForRemoveOperation_BadSignature() throws NoSuchAlgorithmException { KeyPair senderKeys = TestUtils.generateKeyPair(); KeyPair receiverKeys = TestUtils.generateKeyPair(); MailboxStoragePayload mailboxStoragePayload = buildMailboxStoragePayload(senderKeys.getPublic(), receiverKeys.getPublic()); - ProtectedStorageEntry protectedStorageEntry = buildProtectedMailboxStorageEntry(mailboxStoragePayload, receiverKeys, receiverKeys.getPublic(), 1); - - protectedStorageEntry.updateSignature(new byte[] { 0 }); + ProtectedStorageEntry protectedStorageEntry = + new ProtectedMailboxStorageEntry(mailboxStoragePayload, receiverKeys.getPublic(), + 1, new byte[] { 0 }, receiverKeys.getPublic(), Clock.systemDefaultZone()); Assert.assertFalse(protectedStorageEntry.isValidForRemoveOperation()); } diff --git a/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java index 1b5313977e7..b1e9727f032 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java @@ -125,11 +125,13 @@ public void isValidForAddOperation_invalidMailboxPayloadReceiver() throws NoSuch // TESTCASE: validForAddOperation() should fail if the signature isn't valid @Test - public void isValidForAddOperation_BadSignature() throws NoSuchAlgorithmException, CryptoException { + public void isValidForAddOperation_BadSignature() throws NoSuchAlgorithmException { KeyPair ownerKeys = TestUtils.generateKeyPair(); - ProtectedStorageEntry protectedStorageEntry = buildProtectedStorageEntry(ownerKeys, ownerKeys, 1); - protectedStorageEntry.updateSignature( new byte[] { 0 }); + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry protectedStorageEntry = + new ProtectedStorageEntry(protectedStoragePayload, ownerKeys.getPublic(), + 1, new byte[] { 0 }, Clock.systemDefaultZone()); Assert.assertFalse(protectedStorageEntry.isValidForAddOperation()); } @@ -181,11 +183,13 @@ public void isValidForRemoveOperation_invalidMailboxPayloadReceiver() throws NoS // TESTCASE: isValidForRemoveOperation() should fail if the signature is bad @Test - public void isValidForRemoveOperation_BadSignature() throws NoSuchAlgorithmException, CryptoException { + public void isValidForRemoveOperation_BadSignature() throws NoSuchAlgorithmException { KeyPair ownerKeys = TestUtils.generateKeyPair(); - ProtectedStorageEntry protectedStorageEntry = buildProtectedStorageEntry(ownerKeys, ownerKeys, 1); - protectedStorageEntry.updateSignature(new byte[] { 0 }); + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry protectedStorageEntry = + new ProtectedStorageEntry(protectedStoragePayload, ownerKeys.getPublic(), + 1, new byte[] { 0 }, Clock.systemDefaultZone()); Assert.assertFalse(protectedStorageEntry.isValidForRemoveOperation()); } From 24ecfc705557e29f937e6f81ca1463f454b51be5 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 23 Nov 2019 13:46:56 -0800 Subject: [PATCH 41/51] Remove ProtectedStorageEntry::maybeAdjustCreationTimeStamp There is only 1 caller can be replaced with Math.min() --- .../storage/payload/ProtectedStorageEntry.java | 10 ++-------- .../payload/ProtectedStorageEntryTest.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java index 87782af57a3..d87c5a176b4 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java @@ -80,9 +80,9 @@ protected ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, this.sequenceNumber = sequenceNumber; this.signature = signature; - this.creationTimeStamp = creationTimeStamp; - maybeAdjustCreationTimeStamp(clock); + // We don't allow creation date in the future, but we cannot be too strict as clocks are not synced + this.creationTimeStamp = Math.min(creationTimeStamp, clock.millis()); } /////////////////////////////////////////////////////////////////////////////////////////// @@ -135,12 +135,6 @@ public static ProtectedStorageEntry fromProto(protobuf.ProtectedStorageEntry pro // API /////////////////////////////////////////////////////////////////////////////////////////// - public void maybeAdjustCreationTimeStamp(Clock clock) { - // We don't allow creation date in the future, but we cannot be too strict as clocks are not synced - if (creationTimeStamp > clock.millis()) - creationTimeStamp = clock.millis(); - } - public void backDate() { if (protectedStoragePayload instanceof ExpirablePayload) creationTimeStamp -= ((ExpirablePayload) protectedStoragePayload).getTTL() / 2; diff --git a/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java b/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java index b1e9727f032..a624095ba49 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/payload/ProtectedStorageEntryTest.java @@ -31,6 +31,7 @@ import java.security.PublicKey; import java.time.Clock; +import java.time.Duration; import org.junit.Assert; import org.junit.Before; @@ -253,4 +254,20 @@ public protobuf.PersistableNetworkPayload toProtoMessage() { new ProtectedStorageEntry(incompatiblePayload,ownerKeys.getPublic(), 1, new byte[] { 0 }, Clock.systemDefaultZone()); } + + // TESTCASE: PSEs received with future-dated timestamps are updated to be min(currentTime, creationTimeStamp) + @Test + public void futureTimestampIsSanitized() throws NoSuchAlgorithmException { + KeyPair ownerKeys = TestUtils.generateKeyPair(); + + Clock baseClock = Clock.systemDefaultZone(); + Clock futureClock = Clock.offset(baseClock, Duration.ofDays(1)); + + ProtectedStoragePayload protectedStoragePayload = new ProtectedStoragePayloadStub(ownerKeys.getPublic()); + ProtectedStorageEntry protectedStorageEntry = + new ProtectedStorageEntry(protectedStoragePayload, Sig.getPublicKeyBytes(ownerKeys.getPublic()), + ownerKeys.getPublic(), 1, new byte[] { 0 }, futureClock.millis(), baseClock); + + Assert.assertTrue(protectedStorageEntry.getCreationTimeStamp() <= baseClock.millis()); + } } From 0c676080b6e967165f9ad6296630d2c1222ae712 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 23 Nov 2019 14:05:17 -0800 Subject: [PATCH 42/51] @NotNull ProtectedStorageEntry::ownerPubKey In proto3 the initialized value is an empty ByteString and there are no valid uses of passing in null here. --- .../payload/ProtectedStorageEntry.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java index d87c5a176b4..db6f15c79fb 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java @@ -39,6 +39,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + @Getter @EqualsAndHashCode @Slf4j @@ -51,10 +53,10 @@ public class ProtectedStorageEntry implements NetworkPayload, PersistablePayload private long creationTimeStamp; public ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, - PublicKey ownerPubKey, - int sequenceNumber, - byte[] signature, - Clock clock) { + @NotNull PublicKey ownerPubKey, + int sequenceNumber, + byte[] signature, + Clock clock) { this(protectedStoragePayload, Sig.getPublicKeyBytes(ownerPubKey), ownerPubKey, @@ -65,12 +67,12 @@ public ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, } protected ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, - byte[] ownerPubKeyBytes, - PublicKey ownerPubKey, - int sequenceNumber, - byte[] signature, - long creationTimeStamp, - Clock clock) { + byte[] ownerPubKeyBytes, + @NotNull PublicKey ownerPubKey, + int sequenceNumber, + byte[] signature, + long creationTimeStamp, + Clock clock) { Preconditions.checkArgument(!(protectedStoragePayload instanceof PersistableNetworkPayload)); @@ -90,11 +92,11 @@ protected ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, /////////////////////////////////////////////////////////////////////////////////////////// private ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, - byte[] ownerPubKeyBytes, - int sequenceNumber, - byte[] signature, - long creationTimeStamp, - Clock clock) { + byte[] ownerPubKeyBytes, + int sequenceNumber, + byte[] signature, + long creationTimeStamp, + Clock clock) { this(protectedStoragePayload, ownerPubKeyBytes, Sig.getPublicKeyFromBytes(ownerPubKeyBytes), @@ -160,8 +162,7 @@ public boolean isValidForAddOperation() { mailboxStoragePayload.getSenderPubKeyForAddOperation().equals(this.getOwnerPubKey()); } else { - boolean result = this.ownerPubKey != null && - this.protectedStoragePayload != null && + boolean result = this.protectedStoragePayload != null && this.ownerPubKey.equals(protectedStoragePayload.getOwnerPubKey()); if (!result) { @@ -228,9 +229,8 @@ public boolean matchesRelevantPubKey(ProtectedStorageEntry protectedStorageEntry boolean result = protectedStorageEntry.getOwnerPubKey().equals(this.ownerPubKey); if (!result) { - log.warn("New data entry does not match our stored data. storedData.ownerPubKey=" + - (protectedStorageEntry.getOwnerPubKey() != null ? protectedStorageEntry.getOwnerPubKey().toString() : "null") + - ", ownerPubKey=" + this.ownerPubKey); + log.warn("New data entry does not match our stored data. storedData.ownerPubKey={}, ownerPubKey={}}", + protectedStorageEntry.getOwnerPubKey().toString(), this.ownerPubKey); } return result; From 76e8c5736a6c5f56cca83323c373d2f3eb3f2542 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 23 Nov 2019 14:16:25 -0800 Subject: [PATCH 43/51] @NotNull ProtectedStorageEntry::protectedStoragePayload The ProtectedStoragePayload.fromProto code will throw an exception if this is null from the wire so there is no valid use for it to be null. --- .../payload/ProtectedMailboxStorageEntry.java | 4 ++-- .../p2p/storage/payload/ProtectedStorageEntry.java | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java index feaa7b19dd6..a513f972b21 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java @@ -107,7 +107,7 @@ public boolean isValidForAddOperation() { if (!result) { String res1 = this.toString(); String res2 = "null"; - if (mailboxStoragePayload != null && mailboxStoragePayload.getOwnerPubKey() != null) + if (mailboxStoragePayload.getOwnerPubKey() != null) res2 = Utilities.encodeToHex(mailboxStoragePayload.getSenderPubKeyForAddOperation().getEncoded(),true); log.warn("ProtectedMailboxStorageEntry::isValidForAddOperation() failed. " + @@ -141,7 +141,7 @@ public boolean isValidForRemoveOperation() { if (!result) { String res1 = this.toString(); String res2 = "null"; - if (mailboxStoragePayload != null && mailboxStoragePayload.getOwnerPubKey() != null) + if (mailboxStoragePayload.getOwnerPubKey() != null) res2 = Utilities.encodeToHex(mailboxStoragePayload.getOwnerPubKey().getEncoded(), true); log.warn("ProtectedMailboxStorageEntry::isValidForRemoveOperation() failed. " + diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java index db6f15c79fb..29d56a248c0 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java @@ -52,7 +52,7 @@ public class ProtectedStorageEntry implements NetworkPayload, PersistablePayload private final byte[] signature; private long creationTimeStamp; - public ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, + public ProtectedStorageEntry(@NotNull ProtectedStoragePayload protectedStoragePayload, @NotNull PublicKey ownerPubKey, int sequenceNumber, byte[] signature, @@ -66,7 +66,7 @@ public ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, clock); } - protected ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, + protected ProtectedStorageEntry(@NotNull ProtectedStoragePayload protectedStoragePayload, byte[] ownerPubKeyBytes, @NotNull PublicKey ownerPubKey, int sequenceNumber, @@ -91,7 +91,7 @@ protected ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, // PROTO BUFFER /////////////////////////////////////////////////////////////////////////////////////////// - private ProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, + private ProtectedStorageEntry(@NotNull ProtectedStoragePayload protectedStoragePayload, byte[] ownerPubKeyBytes, int sequenceNumber, byte[] signature, @@ -162,13 +162,12 @@ public boolean isValidForAddOperation() { mailboxStoragePayload.getSenderPubKeyForAddOperation().equals(this.getOwnerPubKey()); } else { - boolean result = this.protectedStoragePayload != null && - this.ownerPubKey.equals(protectedStoragePayload.getOwnerPubKey()); + boolean result = this.ownerPubKey.equals(protectedStoragePayload.getOwnerPubKey()); if (!result) { String res1 = this.toString(); String res2 = "null"; - if (protectedStoragePayload != null && protectedStoragePayload.getOwnerPubKey() != null) + if (protectedStoragePayload.getOwnerPubKey() != null) res2 = Utilities.encodeToHex(protectedStoragePayload.getOwnerPubKey().getEncoded(), true); log.warn("ProtectedStorageEntry::isValidForAddOperation() failed. Entry owner does not match Payload owner:\n" + @@ -191,7 +190,7 @@ public boolean isValidForRemoveOperation() { if (!result) { String res1 = this.toString(); String res2 = "null"; - if (protectedStoragePayload != null && protectedStoragePayload.getOwnerPubKey() != null) + if (protectedStoragePayload.getOwnerPubKey() != null) res2 = Utilities.encodeToHex(protectedStoragePayload.getOwnerPubKey().getEncoded(), true); log.warn("ProtectedStorageEntry::isValidForRemoveOperation() failed. Entry owner does not match Payload owner:\n" + From 104984cc22e2dd79fcca88b925dd9dada4fe86f2 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 23 Nov 2019 14:33:03 -0800 Subject: [PATCH 44/51] @NotNull MailboxStoragePayload::senderPubKeyForAddOperation In proto3 this is intialized to an empty ByteString so there is no valid use for it to be null. --- .../network/p2p/storage/payload/MailboxStoragePayload.java | 4 +++- .../p2p/storage/payload/ProtectedMailboxStorageEntry.java | 3 +-- .../network/p2p/storage/payload/ProtectedStorageEntry.java | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/MailboxStoragePayload.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/MailboxStoragePayload.java index 7f40c8e9def..dbe8f622316 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/MailboxStoragePayload.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/MailboxStoragePayload.java @@ -37,6 +37,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + import javax.annotation.Nullable; /** @@ -63,7 +65,7 @@ public final class MailboxStoragePayload implements ProtectedStoragePayload, Exp private Map extraDataMap; public MailboxStoragePayload(PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage, - PublicKey senderPubKeyForAddOperation, + @NotNull PublicKey senderPubKeyForAddOperation, PublicKey ownerPubKey) { this.prefixedSealedAndSignedMessage = prefixedSealedAndSignedMessage; this.senderPubKeyForAddOperation = senderPubKeyForAddOperation; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java index a513f972b21..626ff6abea0 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedMailboxStorageEntry.java @@ -101,8 +101,7 @@ public boolean isValidForAddOperation() { return false; } - boolean result = mailboxStoragePayload.getSenderPubKeyForAddOperation() != null && - mailboxStoragePayload.getSenderPubKeyForAddOperation().equals(this.getOwnerPubKey()); + boolean result = mailboxStoragePayload.getSenderPubKeyForAddOperation().equals(this.getOwnerPubKey()); if (!result) { String res1 = this.toString(); diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java index 29d56a248c0..f550330cfa6 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java @@ -158,8 +158,7 @@ public boolean isValidForAddOperation() { // TODO: The code currently supports MailboxStoragePayload objects inside ProtectedStorageEntry. Fix this. if (protectedStoragePayload instanceof MailboxStoragePayload) { MailboxStoragePayload mailboxStoragePayload = (MailboxStoragePayload) this.getProtectedStoragePayload(); - return mailboxStoragePayload.getSenderPubKeyForAddOperation() != null && - mailboxStoragePayload.getSenderPubKeyForAddOperation().equals(this.getOwnerPubKey()); + return mailboxStoragePayload.getSenderPubKeyForAddOperation().equals(this.getOwnerPubKey()); } else { boolean result = this.ownerPubKey.equals(protectedStoragePayload.getOwnerPubKey()); From 01a7f79eec6e8313083ec22f54577bb28850d7e3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 4 Dec 2019 12:15:53 -0800 Subject: [PATCH 45/51] Make CHECK_TTL_INTERVAL_SEC final It is never changed --- p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 9371541fa1c..1763be24fdb 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -116,7 +116,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers public static final int PURGE_AGE_DAYS = 10; @VisibleForTesting - public static int CHECK_TTL_INTERVAL_SEC = 60; + public static final int CHECK_TTL_INTERVAL_SEC = 60; private boolean initialRequestApplied = false; From c38ff9bf95ec02dd4857a255acc3a0437a39dec8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 4 Dec 2019 16:01:52 -0800 Subject: [PATCH 46/51] s/networkPayload/protectedStoragePayload Helps readability when the variable name matches the type. --- .../java/bisq/network/p2p/storage/P2PDataStorage.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 1763be24fdb..15516da4926 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -443,16 +443,16 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection if (connection.hasPeersNodeAddress() && !closeConnectionReason.isIntended) { map.values() .forEach(protectedStorageEntry -> { - NetworkPayload networkPayload = protectedStorageEntry.getProtectedStoragePayload(); - if (networkPayload instanceof ExpirablePayload && networkPayload instanceof RequiresOwnerIsOnlinePayload) { - NodeAddress ownerNodeAddress = ((RequiresOwnerIsOnlinePayload) networkPayload).getOwnerNodeAddress(); + ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); + if (protectedStoragePayload instanceof ExpirablePayload && protectedStoragePayload instanceof RequiresOwnerIsOnlinePayload) { + NodeAddress ownerNodeAddress = ((RequiresOwnerIsOnlinePayload) protectedStoragePayload).getOwnerNodeAddress(); if (connection.getPeersNodeAddressOptional().isPresent() && ownerNodeAddress.equals(connection.getPeersNodeAddressOptional().get())) { // We have a RequiresLiveOwnerData data object with the node address of the // disconnected peer. We remove that data from our map. // Check if we have the data (e.g. OfferPayload) - ByteArray hashOfPayload = get32ByteHashAsByteArray(networkPayload); + ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); boolean containsKey = map.containsKey(hashOfPayload); if (containsKey) { log.debug("We remove the data as the data owner got disconnected with " + From 688405bc6d0027df02821d58c217ebc786ea9ba1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 4 Dec 2019 16:52:58 -0800 Subject: [PATCH 47/51] [TESTS] Make onDisconnect tests more robust Before refactoring the function ensure the tests cover all cases. This fixes a bug where the payload ttl was too low in some instances causing backDate to do no work when it should. --- .../storage/P2PDataStoreDisconnectTest.java | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java index 408edac93f8..11f43ee0a45 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java @@ -57,7 +57,7 @@ private static ProtectedStorageEntry populateTestState(TestState testState, ProtectedStoragePayload protectedStoragePayload = new ExpirableProtectedStoragePayloadStub(ownerKeys.getPublic(), ttl); ProtectedStorageEntry protectedStorageEntry = testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, TestState.getTestNodeAddress(), null); + testState.mockedStorage.addProtectedStorageEntry(protectedStorageEntry, getTestNodeAddress(), null); return protectedStorageEntry; } @@ -85,17 +85,30 @@ public void setUp() { // TESTCASE: Bad peer info @Test - public void peerConnectionUnknown() { + public void peerConnectionUnknown() throws CryptoException, NoSuchAlgorithmException { when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(false); + ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); + + SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); + + verifyStateAfterDisconnect(this.testState, beforeState, false, false); } // TESTCASE: Intended disconnects don't trigger expiration @Test - public void connectionClosedIntended() { + public void connectionClosedIntended() throws CryptoException, NoSuchAlgorithmException { when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); + + ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); + + SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); + this.testState.mockedStorage.onDisconnect(CloseConnectionReason.CLOSE_REQUESTED_BY_PEER, mockedConnection); + + verifyStateAfterDisconnect(this.testState, beforeState, false, false); } // TESTCASE: Peer NodeAddress unknown @@ -104,7 +117,7 @@ public void connectionClosedSkipsItemsPeerInfoBadState() throws NoSuchAlgorithmE when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.empty()); - ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 1); + ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); @@ -134,7 +147,7 @@ public void connectionClosedSkipsItemsNotFromPeer() throws NoSuchAlgorithmExcept when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(new NodeAddress("notTestNode", 2020))); - ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 1); + ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); @@ -149,7 +162,7 @@ public void connectionClosedReduceTTLAndExpireItemsFromPeer() throws NoSuchAlgor when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(TestState.getTestNodeAddress())); - ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 1); + ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); @@ -158,7 +171,7 @@ public void connectionClosedReduceTTLAndExpireItemsFromPeer() throws NoSuchAlgor this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - verifyStateAfterDisconnect(this.testState, beforeState, true, false); + verifyStateAfterDisconnect(this.testState, beforeState, true, true); } // TESTCASE: ProtectedStoragePayloads implementing the PersistablePayload interface are correctly removed @@ -186,7 +199,7 @@ private ExpirablePersistentProtectedStoragePayloadStub(PublicKey ownerPubKey) { testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); testState.mockedStorage.addProtectedStorageEntry( - protectedStorageEntry, TestState.getTestNodeAddress(), null); + protectedStorageEntry, getTestNodeAddress(), null); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); From df2e4cc0132f5aa8b28ae640962bdbfd9366594d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 4 Dec 2019 17:04:05 -0800 Subject: [PATCH 48/51] Refactor P2PDataStorage::onDisconnect 1. Remove delete during stream iteration 2. Minimize branching w/ early returns for bad states 3. Use stream filter for readability 4. Implement additional checks that should be done when removing entries --- .../network/p2p/storage/P2PDataStorage.java | 85 ++++++++++--------- .../storage/P2PDataStoreDisconnectTest.java | 21 ++--- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 15516da4926..9b14f508c01 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -440,46 +440,51 @@ public void onConnection(Connection connection) { @Override public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { - if (connection.hasPeersNodeAddress() && !closeConnectionReason.isIntended) { - map.values() - .forEach(protectedStorageEntry -> { - ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload(); - if (protectedStoragePayload instanceof ExpirablePayload && protectedStoragePayload instanceof RequiresOwnerIsOnlinePayload) { - NodeAddress ownerNodeAddress = ((RequiresOwnerIsOnlinePayload) protectedStoragePayload).getOwnerNodeAddress(); - if (connection.getPeersNodeAddressOptional().isPresent() && - ownerNodeAddress.equals(connection.getPeersNodeAddressOptional().get())) { - // We have a RequiresLiveOwnerData data object with the node address of the - // disconnected peer. We remove that data from our map. - - // Check if we have the data (e.g. OfferPayload) - ByteArray hashOfPayload = get32ByteHashAsByteArray(protectedStoragePayload); - boolean containsKey = map.containsKey(hashOfPayload); - if (containsKey) { - log.debug("We remove the data as the data owner got disconnected with " + - "closeConnectionReason=" + closeConnectionReason); - - // We only set the data back by half of the TTL and remove the data only if is has - // expired after that back dating. - // We might get connection drops which are not caused by the node going offline, so - // we give more tolerance with that approach, giving the node the change to - // refresh the TTL with a refresh message. - // We observed those issues during stress tests, but it might have been caused by the - // test set up (many nodes/connections over 1 router) - // TODO investigate what causes the disconnections. - // Usually the are: SOCKET_TIMEOUT ,TERMINATED (EOFException) - protectedStorageEntry.backDate(); - if (protectedStorageEntry.isExpired(this.clock)) { - log.info("We found an expired data entry which we have already back dated. " + - "We remove the protectedStoragePayload:\n\t" + Utilities.toTruncatedString(protectedStorageEntry.getProtectedStoragePayload(), 100)); - removeFromMapAndDataStore(protectedStorageEntry, hashOfPayload); - } - } else { - log.debug("Remove data ignored as we don't have an entry for that data."); - } - } - } - }); - } + if (closeConnectionReason.isIntended) + return; + + if (!connection.getPeersNodeAddressOptional().isPresent()) + return; + + NodeAddress peersNodeAddress = connection.getPeersNodeAddressOptional().get(); + + // Retrieve all the eligible payloads based on the node that disconnected + ArrayList> toBackDate = + map.entrySet().stream() + .filter(entry -> entry.getValue().getProtectedStoragePayload() instanceof ExpirablePayload) + .filter(entry -> entry.getValue().getProtectedStoragePayload() instanceof RequiresOwnerIsOnlinePayload) + .filter(entry -> ((RequiresOwnerIsOnlinePayload) entry.getValue().getProtectedStoragePayload()).getOwnerNodeAddress().equals(peersNodeAddress)) + .collect(Collectors.toCollection(ArrayList::new)); + + // Backdate each payload + toBackDate.forEach(mapEntry -> { + // We only set the data back by half of the TTL and remove the data only if is has + // expired after that back dating. + // We might get connection drops which are not caused by the node going offline, so + // we give more tolerance with that approach, giving the node the change to + // refresh the TTL with a refresh message. + // We observed those issues during stress tests, but it might have been caused by the + // test set up (many nodes/connections over 1 router) + // TODO investigate what causes the disconnections. + // Usually the are: SOCKET_TIMEOUT ,TERMINATED (EOFException) + log.debug("We remove the data as the data owner got disconnected with " + + "closeConnectionReason=" + closeConnectionReason); + mapEntry.getValue().backDate(); + }); + + // Remove each backdated payload that is now expired + ArrayList> toRemoveList = + toBackDate.stream().filter(mapEntry -> mapEntry.getValue().isExpired(this.clock)) + .collect(Collectors.toCollection(ArrayList::new)); + + toRemoveList.forEach(toRemoveItem -> { + log.debug("We found an expired data entry. We remove the protectedData:\n\t" + + Utilities.toTruncatedString(toRemoveItem.getValue())); + }); + removeFromMapAndDataStore(toRemoveList); + + if (sequenceNumberMap.size() > this.maxSequenceNumberMapSizeBeforePurge) + sequenceNumberMap.setMap(getPurgedSequenceNumberMap(sequenceNumberMap.getMap())); } @Override diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java index 11f43ee0a45..4d9deb1bf58 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java @@ -86,8 +86,7 @@ public void setUp() { // TESTCASE: Bad peer info @Test public void peerConnectionUnknown() throws CryptoException, NoSuchAlgorithmException { - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(false); - + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.empty()); ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); @@ -100,8 +99,7 @@ public void peerConnectionUnknown() throws CryptoException, NoSuchAlgorithmExcep // TESTCASE: Intended disconnects don't trigger expiration @Test public void connectionClosedIntended() throws CryptoException, NoSuchAlgorithmException { - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); - + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(getTestNodeAddress())); ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); @@ -114,8 +112,7 @@ public void connectionClosedIntended() throws CryptoException, NoSuchAlgorithmEx // TESTCASE: Peer NodeAddress unknown @Test public void connectionClosedSkipsItemsPeerInfoBadState() throws NoSuchAlgorithmException, CryptoException { - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); - when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.empty()); + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.empty()); ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); @@ -129,8 +126,7 @@ public void connectionClosedSkipsItemsPeerInfoBadState() throws NoSuchAlgorithmE // TESTCASE: Unintended disconnects reduce the TTL for entrys that match disconnected peer @Test public void connectionClosedReduceTTL() throws NoSuchAlgorithmException, CryptoException { - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); - when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(TestState.getTestNodeAddress())); + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(getTestNodeAddress())); ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, TimeUnit.DAYS.toMillis(90)); @@ -144,8 +140,7 @@ public void connectionClosedReduceTTL() throws NoSuchAlgorithmException, CryptoE // TESTCASE: Unintended disconnects don't reduce TTL for entrys that are not from disconnected peer @Test public void connectionClosedSkipsItemsNotFromPeer() throws NoSuchAlgorithmException, CryptoException { - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); - when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(new NodeAddress("notTestNode", 2020))); + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(new NodeAddress("notTestNode", 2020))); ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); @@ -159,8 +154,7 @@ public void connectionClosedSkipsItemsNotFromPeer() throws NoSuchAlgorithmExcept // TESTCASE: Unintended disconnects expire entrys that match disconnected peer and TTL is low enough for expire @Test public void connectionClosedReduceTTLAndExpireItemsFromPeer() throws NoSuchAlgorithmException, CryptoException { - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); - when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(TestState.getTestNodeAddress())); + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(getTestNodeAddress())); ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); @@ -188,8 +182,7 @@ private ExpirablePersistentProtectedStoragePayloadStub(PublicKey ownerPubKey) { } } - when(this.mockedConnection.hasPeersNodeAddress()).thenReturn(true); - when(mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(TestState.getTestNodeAddress())); + when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(getTestNodeAddress())); KeyPair ownerKeys = TestUtils.generateKeyPair(); ProtectedStoragePayload protectedStoragePayload = From b16600939825189ee6c3e84e057e996991277f66 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 4 Dec 2019 17:20:19 -0800 Subject: [PATCH 49/51] Remove expire optimization in onDisconnect We already have a garbage collection thread that runs every minute to clean up items. Doing it again during onDisconnect is an unnecessary optimization that adds complexity and caused bugs. For example, the original implementation did not handle the sequence number map correctly and was removing entries during a stream iteration. This also reduces the complexity of testing. There is one code path responsible for reducing ttls and one code path responsible for expiring entries. Much easier to reason about. --- .../network/p2p/storage/P2PDataStorage.java | 55 +++++---------- .../storage/P2PDataStoreDisconnectTest.java | 68 ++----------------- 2 files changed, 24 insertions(+), 99 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 9b14f508c01..b24a5bbd68a 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -448,43 +448,24 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection NodeAddress peersNodeAddress = connection.getPeersNodeAddressOptional().get(); - // Retrieve all the eligible payloads based on the node that disconnected - ArrayList> toBackDate = - map.entrySet().stream() - .filter(entry -> entry.getValue().getProtectedStoragePayload() instanceof ExpirablePayload) - .filter(entry -> entry.getValue().getProtectedStoragePayload() instanceof RequiresOwnerIsOnlinePayload) - .filter(entry -> ((RequiresOwnerIsOnlinePayload) entry.getValue().getProtectedStoragePayload()).getOwnerNodeAddress().equals(peersNodeAddress)) - .collect(Collectors.toCollection(ArrayList::new)); - - // Backdate each payload - toBackDate.forEach(mapEntry -> { - // We only set the data back by half of the TTL and remove the data only if is has - // expired after that back dating. - // We might get connection drops which are not caused by the node going offline, so - // we give more tolerance with that approach, giving the node the change to - // refresh the TTL with a refresh message. - // We observed those issues during stress tests, but it might have been caused by the - // test set up (many nodes/connections over 1 router) - // TODO investigate what causes the disconnections. - // Usually the are: SOCKET_TIMEOUT ,TERMINATED (EOFException) - log.debug("We remove the data as the data owner got disconnected with " + - "closeConnectionReason=" + closeConnectionReason); - mapEntry.getValue().backDate(); - }); - - // Remove each backdated payload that is now expired - ArrayList> toRemoveList = - toBackDate.stream().filter(mapEntry -> mapEntry.getValue().isExpired(this.clock)) - .collect(Collectors.toCollection(ArrayList::new)); - - toRemoveList.forEach(toRemoveItem -> { - log.debug("We found an expired data entry. We remove the protectedData:\n\t" + - Utilities.toTruncatedString(toRemoveItem.getValue())); - }); - removeFromMapAndDataStore(toRemoveList); - - if (sequenceNumberMap.size() > this.maxSequenceNumberMapSizeBeforePurge) - sequenceNumberMap.setMap(getPurgedSequenceNumberMap(sequenceNumberMap.getMap())); + // Backdate all the eligible payloads based on the node that disconnected + map.values().stream() + .filter(protectedStorageEntry -> protectedStorageEntry.getProtectedStoragePayload() instanceof ExpirablePayload) + .filter(protectedStorageEntry -> protectedStorageEntry.getProtectedStoragePayload() instanceof RequiresOwnerIsOnlinePayload) + .filter(protectedStorageEntry -> ((RequiresOwnerIsOnlinePayload) protectedStorageEntry.getProtectedStoragePayload()).getOwnerNodeAddress().equals(peersNodeAddress)) + .forEach(protectedStorageEntry -> { + // We only set the data back by half of the TTL and remove the data only if is has + // expired after that back dating. + // We might get connection drops which are not caused by the node going offline, so + // we give more tolerance with that approach, giving the node the change to + // refresh the TTL with a refresh message. + // We observed those issues during stress tests, but it might have been caused by the + // test set up (many nodes/connections over 1 router) + // TODO investigate what causes the disconnections. + // Usually the are: SOCKET_TIMEOUT ,TERMINATED (EOFException) + log.debug("Backdating {} due to closeConnectionReason={}", protectedStorageEntry, closeConnectionReason); + protectedStorageEntry.backDate(); + }); } @Override diff --git a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java index 4d9deb1bf58..18cd2a21fc9 100644 --- a/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java +++ b/p2p/src/test/java/bisq/network/p2p/storage/P2PDataStoreDisconnectTest.java @@ -26,11 +26,9 @@ import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.crypto.CryptoException; -import bisq.common.proto.persistable.PersistablePayload; import java.security.KeyPair; import java.security.NoSuchAlgorithmException; -import java.security.PublicKey; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -64,12 +62,11 @@ private static ProtectedStorageEntry populateTestState(TestState testState, private static void verifyStateAfterDisconnect(TestState currentState, SavedTestState beforeState, - boolean wasRemoved, boolean wasTTLReduced) { ProtectedStorageEntry protectedStorageEntry = beforeState.protectedStorageEntryBeforeOp; currentState.verifyProtectedStorageRemove(beforeState, protectedStorageEntry, - wasRemoved, wasRemoved, false, false); + false, false, false, false); if (wasTTLReduced) Assert.assertTrue(protectedStorageEntry.getCreationTimeStamp() < beforeState.creationTimestampBeforeUpdate); @@ -93,7 +90,7 @@ public void peerConnectionUnknown() throws CryptoException, NoSuchAlgorithmExcep this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - verifyStateAfterDisconnect(this.testState, beforeState, false, false); + verifyStateAfterDisconnect(this.testState, beforeState, false); } // TESTCASE: Intended disconnects don't trigger expiration @@ -106,7 +103,7 @@ public void connectionClosedIntended() throws CryptoException, NoSuchAlgorithmEx this.testState.mockedStorage.onDisconnect(CloseConnectionReason.CLOSE_REQUESTED_BY_PEER, mockedConnection); - verifyStateAfterDisconnect(this.testState, beforeState, false, false); + verifyStateAfterDisconnect(this.testState, beforeState, false); } // TESTCASE: Peer NodeAddress unknown @@ -120,7 +117,7 @@ public void connectionClosedSkipsItemsPeerInfoBadState() throws NoSuchAlgorithmE this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - verifyStateAfterDisconnect(this.testState, beforeState, false, false); + verifyStateAfterDisconnect(this.testState, beforeState, false); } // TESTCASE: Unintended disconnects reduce the TTL for entrys that match disconnected peer @@ -134,7 +131,7 @@ public void connectionClosedReduceTTL() throws NoSuchAlgorithmException, CryptoE this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - verifyStateAfterDisconnect(this.testState, beforeState, false, true); + verifyStateAfterDisconnect(this.testState, beforeState, true); } // TESTCASE: Unintended disconnects don't reduce TTL for entrys that are not from disconnected peer @@ -148,59 +145,6 @@ public void connectionClosedSkipsItemsNotFromPeer() throws NoSuchAlgorithmExcept this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - verifyStateAfterDisconnect(this.testState, beforeState, false, false); - } - - // TESTCASE: Unintended disconnects expire entrys that match disconnected peer and TTL is low enough for expire - @Test - public void connectionClosedReduceTTLAndExpireItemsFromPeer() throws NoSuchAlgorithmException, CryptoException { - when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(getTestNodeAddress())); - - ProtectedStorageEntry protectedStorageEntry = populateTestState(testState, 2); - - SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); - - // Increment the time by 1 hour which will put the protectedStorageState outside TTL - this.testState.incrementClock(); - - this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - - verifyStateAfterDisconnect(this.testState, beforeState, true, true); - } - - // TESTCASE: ProtectedStoragePayloads implementing the PersistablePayload interface are correctly removed - // from the persistent store during the onDisconnect path. - @Test - public void connectionClosedReduceTTLAndExpireItemsFromPeerPersistable() - throws NoSuchAlgorithmException, CryptoException { - - class ExpirablePersistentProtectedStoragePayloadStub - extends ExpirableProtectedStoragePayloadStub implements PersistablePayload { - - private ExpirablePersistentProtectedStoragePayloadStub(PublicKey ownerPubKey) { - super(ownerPubKey, 0); - } - } - - when(this.mockedConnection.getPeersNodeAddressOptional()).thenReturn(Optional.of(getTestNodeAddress())); - - KeyPair ownerKeys = TestUtils.generateKeyPair(); - ProtectedStoragePayload protectedStoragePayload = - new ExpirablePersistentProtectedStoragePayloadStub(ownerKeys.getPublic()); - - ProtectedStorageEntry protectedStorageEntry = - testState.mockedStorage.getProtectedStorageEntry(protectedStoragePayload, ownerKeys); - - testState.mockedStorage.addProtectedStorageEntry( - protectedStorageEntry, getTestNodeAddress(), null); - - SavedTestState beforeState = this.testState.saveTestState(protectedStorageEntry); - - // Increment the time by 1 hour which will put the protectedStorageState outside TTL - this.testState.incrementClock(); - - this.testState.mockedStorage.onDisconnect(CloseConnectionReason.SOCKET_CLOSED, mockedConnection); - - verifyStateAfterDisconnect(this.testState, beforeState, true, false); + verifyStateAfterDisconnect(this.testState, beforeState, false); } } From 7b8d346aea9adfa22bbfa1193a4d3d11d910607f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 4 Dec 2019 17:28:17 -0800 Subject: [PATCH 50/51] Remove filter for ExpirablePayload ProtectedStorageEntry::backDate() already handles this --- p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index b24a5bbd68a..0cdb2871630 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -450,7 +450,6 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection // Backdate all the eligible payloads based on the node that disconnected map.values().stream() - .filter(protectedStorageEntry -> protectedStorageEntry.getProtectedStoragePayload() instanceof ExpirablePayload) .filter(protectedStorageEntry -> protectedStorageEntry.getProtectedStoragePayload() instanceof RequiresOwnerIsOnlinePayload) .filter(protectedStorageEntry -> ((RequiresOwnerIsOnlinePayload) protectedStorageEntry.getProtectedStoragePayload()).getOwnerNodeAddress().equals(peersNodeAddress)) .forEach(protectedStorageEntry -> { From e8c822563572671d32743d0e70fd89274c7db6ca Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 9 Dec 2019 09:12:57 -0800 Subject: [PATCH 51/51] [PR COMMENTS] Fix comment typo s/change/chance/ --- p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 0cdb2871630..9683f4ee01d 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -456,7 +456,7 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection // We only set the data back by half of the TTL and remove the data only if is has // expired after that back dating. // We might get connection drops which are not caused by the node going offline, so - // we give more tolerance with that approach, giving the node the change to + // we give more tolerance with that approach, giving the node the chance to // refresh the TTL with a refresh message. // We observed those issues during stress tests, but it might have been caused by the // test set up (many nodes/connections over 1 router)