diff --git a/CHANGELOG.md b/CHANGELOG.md index cbd9f10a1..5e8457301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to AET will be documented in this file. ## Unreleased **List of changes that are finished but not yet released in any final version.** +- [PR-336](https://github.com/Cognifide/aet/pull/336) Cleaner will remove artifacts without reference in suites e.g after rerun one test - [PR-337](https://github.com/Cognifide/aet/pull/337) Added rerun one test or whole suite ## Version 3.0.0 diff --git a/api/datastorage-api/src/main/java/com/cognifide/aet/vs/ArtifactsDAO.java b/api/datastorage-api/src/main/java/com/cognifide/aet/vs/ArtifactsDAO.java index 80b5d4893..8f8044bca 100644 --- a/api/datastorage-api/src/main/java/com/cognifide/aet/vs/ArtifactsDAO.java +++ b/api/datastorage-api/src/main/java/com/cognifide/aet/vs/ArtifactsDAO.java @@ -77,6 +77,12 @@ public interface ArtifactsDAO extends Serializable { */ String getArtifactAsString(DBKey dbKey, String objectID) throws IOException; + /** + * @param dbKey - key with project and company name + * @return Set of all artifacts id contained in database as String set or empty set + */ + Set getArtifactsIds(DBKey dbKey); + /** * @param dbKey - key with project and company name * @param objectID - suite run identificator diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/context/SuiteAggregationCounter.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/context/SuiteAggregationCounter.java index e0c43e9fe..c372718d4 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/context/SuiteAggregationCounter.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/context/SuiteAggregationCounter.java @@ -28,4 +28,5 @@ public SuiteAggregationCounter(int suitesToAggregate) { public int getSuitesToAggregate() { return suitesToAggregate; } + } diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/FetchAllProjectSuitesProcessor.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/FetchAllProjectSuitesProcessor.java index 24cfc07ba..7d7a44f47 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/FetchAllProjectSuitesProcessor.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/FetchAllProjectSuitesProcessor.java @@ -16,6 +16,7 @@ package com.cognifide.aet.cleaner.processors; import com.cognifide.aet.cleaner.context.CleanerContext; +import com.cognifide.aet.cleaner.context.SuiteAggregationCounter; import com.cognifide.aet.cleaner.processors.exchange.AllSuiteVersionsMessageBody; import com.cognifide.aet.communication.api.metadata.Suite; import com.cognifide.aet.vs.DBKey; @@ -73,6 +74,8 @@ public AllSuiteVersionsMessageBody apply(Map.Entry> in }).toList(); exchange.getOut().setBody(body); + exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY, + new SuiteAggregationCounter(allProjectSuites.size())); exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext); } diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/GetMetadataArtifactsProcessor.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/GetMetadataArtifactsProcessor.java index 712b5f643..ed2156c18 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/GetMetadataArtifactsProcessor.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/GetMetadataArtifactsProcessor.java @@ -77,9 +77,7 @@ public void process(Exchange exchange) throws Exception { ReferencedArtifactsMessageBody body = new ReferencedArtifactsMessageBody( messageBody.getData().getName(), messageBody.getDbKey()); - if (messageBody.shouldBeRemoved()) { - body.setArtifactsToRemove(metatadaArtifacts); - } else { + if (messageBody.shouldBeKept()) { body.setArtifactsToKeep(metatadaArtifacts); } diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessor.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessor.java index 34a6a5808..effe11745 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessor.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessor.java @@ -18,7 +18,7 @@ import com.cognifide.aet.cleaner.context.CleanerContext; import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody; import com.cognifide.aet.vs.ArtifactsDAO; -import com.google.common.collect.Sets; +import java.util.Set; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.osgi.service.component.annotations.Component; @@ -34,6 +34,15 @@ public class RemoveArtifactsProcessor implements Processor { @Reference private ArtifactsDAO artifactsDAO; + public RemoveArtifactsProcessor() { + //default constructor + } + + // for unit tests + public RemoveArtifactsProcessor(ArtifactsDAO artifactsDAO) { + this.artifactsDAO = artifactsDAO; + } + @Override @SuppressWarnings("unchecked") public void process(Exchange exchange) throws Exception { @@ -42,8 +51,7 @@ public void process(Exchange exchange) throws Exception { final ReferencedArtifactsMessageBody messageBody = exchange.getIn() .getBody(ReferencedArtifactsMessageBody.class); - final Sets.SetView artifactsToRemove = - Sets.difference(messageBody.getArtifactsToRemove(), messageBody.getArtifactsToKeep()); + Set artifactsToRemove = getArtifactsIdsToRemove(artifactsDAO, messageBody); LOGGER.debug("Artifacts that will be removed: {}", artifactsToRemove); if (!cleanerContext.isDryRun()) { @@ -57,4 +65,11 @@ public void process(Exchange exchange) throws Exception { artifactsToRemove.size(), messageBody.getDbKey(), messageBody.getData()); } } + + static Set getArtifactsIdsToRemove(ArtifactsDAO artifactsDAO, + ReferencedArtifactsMessageBody messageBody) { + Set artifactsToRemove = artifactsDAO.getArtifactsIds(messageBody.getDbKey()); + artifactsToRemove.removeAll(messageBody.getArtifactsToKeep()); + return artifactsToRemove; + } } diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/SuitesRemovePredicateProcessor.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/SuitesRemovePredicateProcessor.java index 6c9605ab0..5996a2c8c 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/SuitesRemovePredicateProcessor.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/SuitesRemovePredicateProcessor.java @@ -64,8 +64,8 @@ public SuiteMessageBody apply(Suite suite) { }).toList(); exchange.getOut().setBody(body); - exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY, - new SuiteAggregationCounter(suiteVersions.size())); + exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY, exchange.getIn() + .getHeader(SuiteAggregationCounter.NAME_KEY, SuiteAggregationCounter.class)); exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext); } } diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/ReferencedArtifactsMessageBody.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/ReferencedArtifactsMessageBody.java index b1a5d9134..6cfc88a61 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/ReferencedArtifactsMessageBody.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/ReferencedArtifactsMessageBody.java @@ -22,9 +22,7 @@ public class ReferencedArtifactsMessageBody extends MessageBody { - private static final long serialVersionUID = 3811933888080393100L; - - private Set artifactsToRemove; + private static final long serialVersionUID = 3748474548512567069L; private Set artifactsToKeep; @@ -36,15 +34,6 @@ public String getId() { return getDbKey().getCompany() + "_" + getDbKey().getProject() + "|" + getData(); } - public Set getArtifactsToRemove() { - return artifactsToRemove != null ? Collections.unmodifiableSet(artifactsToRemove) - : Collections.emptySet(); - } - - public void setArtifactsToRemove(Set artifactsToRemove) { - this.artifactsToRemove = artifactsToRemove; - } - public Set getArtifactsToKeep() { return artifactsToKeep != null ? Collections.unmodifiableSet(artifactsToKeep) : Collections.emptySet(); @@ -55,11 +44,6 @@ public void setArtifactsToKeep(Set artifactsToKeep) { } public void update(ReferencedArtifactsMessageBody body) { - if (artifactsToRemove == null) { - artifactsToRemove = new HashSet<>(); - } - artifactsToRemove.addAll(body.getArtifactsToRemove()); - if (artifactsToKeep == null) { artifactsToKeep = new HashSet<>(); } diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/SuiteMessageBody.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/SuiteMessageBody.java index 81e78f1e8..409d24432 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/SuiteMessageBody.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/SuiteMessageBody.java @@ -32,8 +32,8 @@ public SuiteMessageBody(Suite data, DBKey dbKey, boolean toRemove) { this.toRemove = toRemove; } - public boolean shouldBeRemoved() { - return toRemove; + public boolean shouldBeKept() { + return !toRemove; } public Set getSuiteArtifacts() { diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/MetadataCleanerRouteBuilder.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/MetadataCleanerRouteBuilder.java index 218c0c65b..44b49c163 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/MetadataCleanerRouteBuilder.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/MetadataCleanerRouteBuilder.java @@ -72,7 +72,7 @@ public void configure() throws Exception { .process(suitesRemovePredicateProcessor) .split(body()) .choice() - .when(body().method("shouldBeRemoved").isEqualTo(true)).to(direct("removeMetadata")) + .when(body().method("shouldBeKept").isEqualTo(false)).to(direct("removeMetadata")) .otherwise().to(direct("getMetadataArtifacts")) .endChoice(); @@ -86,7 +86,7 @@ public void configure() throws Exception { .to(direct(AGGREGATE_SUITES_STEP)); from(direct(AGGREGATE_SUITES_STEP)) - .aggregate(body().method("getId"), new SuitesAggregationStrategy()) + .aggregate(body().method("getDbKey"), new SuitesAggregationStrategy()) .completionSize(header(SuiteAggregationCounter.NAME_KEY).method("getSuitesToAggregate")) .completionTimeout(60000L).forceCompletionOnStop() .to(direct("removeArtifacts")); diff --git a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/SuitesAggregationStrategy.java b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/SuitesAggregationStrategy.java index cc7e6f53a..53b8a6b77 100644 --- a/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/SuitesAggregationStrategy.java +++ b/core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/SuitesAggregationStrategy.java @@ -57,6 +57,6 @@ private boolean isFirstAggregation(Exchange oldExchange) { public void onCompletion(Exchange exchange) { ReferencedArtifactsMessageBody body = exchange.getIn() .getBody(ReferencedArtifactsMessageBody.class); - LOGGER.debug("Finished aggregating {}", body.getId()); + LOGGER.debug("Finished aggregating {}", body.getDbKey()); } } diff --git a/core/cleaner/src/test/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessorTest.java b/core/cleaner/src/test/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessorTest.java new file mode 100644 index 000000000..3ba34cbbf --- /dev/null +++ b/core/cleaner/src/test/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessorTest.java @@ -0,0 +1,157 @@ +/** + * AET + * + * Copyright (C) 2013 Cognifide Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.cognifide.aet.cleaner.processors; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.cognifide.aet.cleaner.context.CleanerContext; +import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody; +import com.cognifide.aet.vs.ArtifactsDAO; +import com.cognifide.aet.vs.DBKey; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.impl.DefaultCamelContext; +import org.apache.camel.impl.DefaultExchange; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class RemoveArtifactsProcessorTest { + + private static final Set ONE_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays + .asList("1", "2", "3", "4", "5", "6", "7")); + + private static final Set ONE_TO_FIVE_ARTIFACTS_ID_SET = new HashSet<>(Arrays + .asList("1", "2", "3", "4", "5")); + + private static final Set SIX_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays + .asList("6", "7")); + + private Exchange exchange; + + @Mock + private ArtifactsDAO artifactDAO; + + @Mock + private CleanerContext cleanerContext; + + @Mock + private DBKey dbKey; + + private RemoveArtifactsProcessor removeArtifactsProcessor; + + @Before + public void setUp() { + when(cleanerContext.isDryRun()).thenReturn(false); + + CamelContext ctx = new DefaultCamelContext(); + exchange = new DefaultExchange(ctx); + + Message in = exchange.getIn(); + in.setBody(new ReferencedArtifactsMessageBody("", dbKey)); + in.setHeader(CleanerContext.KEY_NAME, cleanerContext); + } + + @Test + public void checkIfRemoveArtifactsWasCalled_whenDryRunIsFalse_expectTrue() throws Exception { + when(artifactDAO.getArtifactsIds(any(DBKey.class))) + .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET)); + setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET); + + removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO); + removeArtifactsProcessor.process(exchange); + + verify(artifactDAO, times(1)).removeArtifacts(any(DBKey.class), any(Set.class)); + verify(artifactDAO, times(1)).getArtifactsIds(any(DBKey.class)); + } + + @Test + public void checkIfRemoveArtifactsWasCalled_whenDryRunIsTrue_expectFalse() throws Exception { + when(cleanerContext.isDryRun()).thenReturn(true); + when(artifactDAO.getArtifactsIds(any(DBKey.class))) + .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET)); + setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET); + + removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO); + removeArtifactsProcessor.process(exchange); + + verify(artifactDAO, times(0)).removeArtifacts(any(DBKey.class), any(Set.class)); + verify(artifactDAO, times(1)).getArtifactsIds(any(DBKey.class)); + } + + @Test + public void check_substractArtifactsSets_expectSetOfTwoVariables() throws Exception { + when(artifactDAO.getArtifactsIds(any(DBKey.class))) + .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET)); + setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET); + ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn() + .getBody(); + assertEquals(SIX_TO_SEVEN_ARTIFACTS_ID_SET, + removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody)); + } + + @Test + public void check_substractArtifactsSets_expectEmptySet() throws Exception { + when(artifactDAO.getArtifactsIds(any(DBKey.class))) + .thenReturn(new HashSet<>(ONE_TO_FIVE_ARTIFACTS_ID_SET)); + setArtifactsIdToKeep(ONE_TO_SEVEN_ARTIFACTS_ID_SET); + ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn() + .getBody(); + assertTrue(removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody).isEmpty()); + } + + @Test + public void check_substractArtifactsSets_expectSetOfFiveVariables() throws Exception { + when(artifactDAO.getArtifactsIds(any(DBKey.class))) + .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET)); + setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET); + + ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn() + .getBody(); + + assertEquals(ONE_TO_FIVE_ARTIFACTS_ID_SET, + removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody)); + } + + @Test + public void check_substractArtifactsSetsWhenDbIsEmpty_expectEmptySet() throws Exception { + when(artifactDAO.getArtifactsIds(any(DBKey.class))) + .thenReturn(new HashSet<>()); + setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET); + ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn() + .getBody(); + assertTrue(removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody).isEmpty()); + } + + private void setArtifactsIdToKeep(Set artifactsIdToKeep) { + ReferencedArtifactsMessageBody body = (ReferencedArtifactsMessageBody) exchange.getIn() + .getBody(); + body.setArtifactsToKeep(artifactsIdToKeep); + } + +} diff --git a/core/datastorage/src/main/java/com/cognifide/aet/vs/artifacts/ArtifactsDAOMongoDBImpl.java b/core/datastorage/src/main/java/com/cognifide/aet/vs/artifacts/ArtifactsDAOMongoDBImpl.java index 4ca483e6c..6c3afba8a 100644 --- a/core/datastorage/src/main/java/com/cognifide/aet/vs/artifacts/ArtifactsDAOMongoDBImpl.java +++ b/core/datastorage/src/main/java/com/cognifide/aet/vs/artifacts/ArtifactsDAOMongoDBImpl.java @@ -15,6 +15,8 @@ */ package com.cognifide.aet.vs.artifacts; +import static com.mongodb.client.model.Projections.fields; +import static com.mongodb.client.model.Projections.include; import com.cognifide.aet.vs.Artifact; import com.cognifide.aet.vs.ArtifactsDAO; import com.cognifide.aet.vs.DBKey; @@ -30,6 +32,7 @@ import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.util.Date; +import java.util.HashSet; import java.util.Set; import org.apache.commons.io.IOUtils; import org.bson.Document; @@ -103,6 +106,22 @@ public Artifact getArtifact(DBKey dbKey, String objectID) { return artifact; } + @Override + public Set getArtifactsIds(DBKey dbKey) { + final String dbName = MongoDBClient.getDbName(dbKey.getCompany(), dbKey.getProject()); + FindIterable findIterable = + client.getDatabase(dbName) + .getCollection(ARTIFACTS_COLLECTION_NAME + FILES_COLLECTION_SUFFIX) + .find().projection(fields(include(ID_FIELD_NAME))); + + Set artifactsId = new HashSet<>(); + for (Document document : findIterable) { + artifactsId.add(document.get(ID_FIELD_NAME).toString()); + } + return artifactsId; + } + + @Override public String getArtifactAsString(DBKey dbKey, String objectID) throws IOException { String result = null; diff --git a/core/jobs/src/test/java/com/cognifide/aet/job/common/ArtifactDAOMock.java b/core/jobs/src/test/java/com/cognifide/aet/job/common/ArtifactDAOMock.java index 30db66677..f5484c22a 100644 --- a/core/jobs/src/test/java/com/cognifide/aet/job/common/ArtifactDAOMock.java +++ b/core/jobs/src/test/java/com/cognifide/aet/job/common/ArtifactDAOMock.java @@ -27,6 +27,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Set; @@ -113,6 +114,11 @@ public void removeArtifacts(DBKey dbKey, Set artifactsToRemove) { //do nothing here } + @Override + public Set getArtifactsIds(DBKey dbKey) { + return Collections.emptySet(); + } + /** * workaround to get data for verification * diff --git a/documentation/src/main/wiki/assets/diagrams/aet-cleaner-workflow.png b/documentation/src/main/wiki/assets/diagrams/aet-cleaner-workflow.png index ae6ff62d8..f5df9f3d1 100644 Binary files a/documentation/src/main/wiki/assets/diagrams/aet-cleaner-workflow.png and b/documentation/src/main/wiki/assets/diagrams/aet-cleaner-workflow.png differ