Cleaner will remove artifacts without reference in suites e.g after rerun one test

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to AET will be documented in this file.
 ## Unreleased
 **List of changes that are finished but not yet released in any final version.**
 
+- [PR-336](https://github.com/Cognifide/aet/pull/336) Cleaner will remove artifacts without reference in suites e.g after rerun one test 
 - [PR-337](https://github.com/Cognifide/aet/pull/337) Added rerun one test or whole suite
 
 ## Version 3.0.0

api/datastorage-api/src/main/java/com/cognifide/aet/vs/ArtifactsDAO.java

@@ -77,6 +77,12 @@ public interface ArtifactsDAO extends Serializable {
    */
   String getArtifactAsString(DBKey dbKey, String objectID) throws IOException;
 
+  /**
+   * @param dbKey - key with project and company name
+   * @return Set of all artifacts id contained in database as String set or empty set
+   */
+  Set<String> getArtifactsIds(DBKey dbKey);
+
   /**
    * @param dbKey - key with project and company name
    * @param objectID - suite run identificator

core/cleaner/src/main/java/com/cognifide/aet/cleaner/context/SuiteAggregationCounter.java

@@ -28,4 +28,5 @@ public SuiteAggregationCounter(int suitesToAggregate) {
   public int getSuitesToAggregate() {
     return suitesToAggregate;
   }
+
 }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/FetchAllProjectSuitesProcessor.java

@@ -16,6 +16,7 @@
 package com.cognifide.aet.cleaner.processors;
 
 import com.cognifide.aet.cleaner.context.CleanerContext;
+import com.cognifide.aet.cleaner.context.SuiteAggregationCounter;
 import com.cognifide.aet.cleaner.processors.exchange.AllSuiteVersionsMessageBody;
 import com.cognifide.aet.communication.api.metadata.Suite;
 import com.cognifide.aet.vs.DBKey;
@@ -73,6 +74,8 @@ public AllSuiteVersionsMessageBody apply(Map.Entry<String, Collection<Suite>> in
             }).toList();
 
     exchange.getOut().setBody(body);
+    exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY,
+        new SuiteAggregationCounter(allProjectSuites.size()));
     exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext);
   }
 

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/GetMetadataArtifactsProcessor.java

@@ -77,9 +77,7 @@ public void process(Exchange exchange) throws Exception {
     ReferencedArtifactsMessageBody body = new ReferencedArtifactsMessageBody(
         messageBody.getData().getName(),
         messageBody.getDbKey());
-    if (messageBody.shouldBeRemoved()) {
-      body.setArtifactsToRemove(metatadaArtifacts);
-    } else {
+    if (messageBody.shouldBeKept()) {
       body.setArtifactsToKeep(metatadaArtifacts);
     }
 

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessor.java

@@ -18,7 +18,7 @@
 import com.cognifide.aet.cleaner.context.CleanerContext;
 import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody;
 import com.cognifide.aet.vs.ArtifactsDAO;
-import com.google.common.collect.Sets;
+import java.util.Set;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
 import org.osgi.service.component.annotations.Component;
@@ -34,6 +34,15 @@ public class RemoveArtifactsProcessor implements Processor {
   @Reference
   private ArtifactsDAO artifactsDAO;
 
+  public RemoveArtifactsProcessor() {
+    //default constructor
+  }
+
+  // for unit tests
+  public RemoveArtifactsProcessor(ArtifactsDAO artifactsDAO) {
+    this.artifactsDAO = artifactsDAO;
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   public void process(Exchange exchange) throws Exception {
@@ -42,8 +51,7 @@ public void process(Exchange exchange) throws Exception {
     final ReferencedArtifactsMessageBody messageBody = exchange.getIn()
         .getBody(ReferencedArtifactsMessageBody.class);
 
-    final Sets.SetView<String> artifactsToRemove =
-        Sets.difference(messageBody.getArtifactsToRemove(), messageBody.getArtifactsToKeep());
+    Set<String> artifactsToRemove = getArtifactsIdsToRemove(artifactsDAO, messageBody);
 
     LOGGER.debug("Artifacts that will be removed: {}", artifactsToRemove);
     if (!cleanerContext.isDryRun()) {
@@ -57,4 +65,11 @@ public void process(Exchange exchange) throws Exception {
           artifactsToRemove.size(), messageBody.getDbKey(), messageBody.getData());
     }
   }
+
+  static Set<String> getArtifactsIdsToRemove(ArtifactsDAO artifactsDAO,
+      ReferencedArtifactsMessageBody messageBody) {
+    Set<String> artifactsToRemove = artifactsDAO.getArtifactsIds(messageBody.getDbKey());
+    artifactsToRemove.removeAll(messageBody.getArtifactsToKeep());
+    return artifactsToRemove;
+  }
 }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/SuitesRemovePredicateProcessor.java

@@ -64,8 +64,8 @@ public SuiteMessageBody apply(Suite suite) {
         }).toList();
 
     exchange.getOut().setBody(body);
-    exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY,
-        new SuiteAggregationCounter(suiteVersions.size()));
+    exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY, exchange.getIn()
+        .getHeader(SuiteAggregationCounter.NAME_KEY, SuiteAggregationCounter.class));
     exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext);
   }
 }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/ReferencedArtifactsMessageBody.java

@@ -22,9 +22,7 @@
 
 public class ReferencedArtifactsMessageBody extends MessageBody<String> {
 
-  private static final long serialVersionUID = 3811933888080393100L;
-
-  private Set<String> artifactsToRemove;
+  private static final long serialVersionUID = 3748474548512567069L;
 
   private Set<String> artifactsToKeep;
 
@@ -36,15 +34,6 @@ public String getId() {
     return getDbKey().getCompany() + "_" + getDbKey().getProject() + "|" + getData();
   }
 
-  public Set<String> getArtifactsToRemove() {
-    return artifactsToRemove != null ? Collections.unmodifiableSet(artifactsToRemove)
-        : Collections.<String>emptySet();
-  }
-
-  public void setArtifactsToRemove(Set<String> artifactsToRemove) {
-    this.artifactsToRemove = artifactsToRemove;
-  }
-
   public Set<String> getArtifactsToKeep() {
     return artifactsToKeep != null ? Collections.unmodifiableSet(artifactsToKeep)
         : Collections.<String>emptySet();
@@ -55,11 +44,6 @@ public void setArtifactsToKeep(Set<String> artifactsToKeep) {
   }
 
   public void update(ReferencedArtifactsMessageBody body) {
-    if (artifactsToRemove == null) {
-      artifactsToRemove = new HashSet<>();
-    }
-    artifactsToRemove.addAll(body.getArtifactsToRemove());
-
     if (artifactsToKeep == null) {
       artifactsToKeep = new HashSet<>();
     }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/SuiteMessageBody.java

@@ -32,8 +32,8 @@ public SuiteMessageBody(Suite data, DBKey dbKey, boolean toRemove) {
     this.toRemove = toRemove;
   }
 
-  public boolean shouldBeRemoved() {
-    return toRemove;
+  public boolean shouldBeKept() {
+    return !toRemove;
   }
 
   public Set<String> getSuiteArtifacts() {

core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/MetadataCleanerRouteBuilder.java

@@ -72,7 +72,7 @@ public void configure() throws Exception {
         .process(suitesRemovePredicateProcessor)
         .split(body())
         .choice()
-        .when(body().method("shouldBeRemoved").isEqualTo(true)).to(direct("removeMetadata"))
+        .when(body().method("shouldBeKept").isEqualTo(false)).to(direct("removeMetadata"))
         .otherwise().to(direct("getMetadataArtifacts"))
         .endChoice();
 
@@ -86,7 +86,7 @@ public void configure() throws Exception {
         .to(direct(AGGREGATE_SUITES_STEP));
 
     from(direct(AGGREGATE_SUITES_STEP))
-        .aggregate(body().method("getId"), new SuitesAggregationStrategy())
+        .aggregate(body().method("getDbKey"), new SuitesAggregationStrategy())
         .completionSize(header(SuiteAggregationCounter.NAME_KEY).method("getSuitesToAggregate"))
         .completionTimeout(60000L).forceCompletionOnStop()
         .to(direct("removeArtifacts"));

core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/SuitesAggregationStrategy.java

@@ -57,6 +57,6 @@ private boolean isFirstAggregation(Exchange oldExchange) {
   public void onCompletion(Exchange exchange) {
     ReferencedArtifactsMessageBody body = exchange.getIn()
         .getBody(ReferencedArtifactsMessageBody.class);
-    LOGGER.debug("Finished aggregating {}", body.getId());
+    LOGGER.debug("Finished aggregating {}", body.getDbKey());
   }
 }

core/cleaner/src/test/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessorTest.java

@@ -0,0 +1,157 @@
+/**
+ * AET
+ *
+ * Copyright (C) 2013 Cognifide Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.cognifide.aet.cleaner.processors;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.cognifide.aet.cleaner.context.CleanerContext;
+import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody;
+import com.cognifide.aet.vs.ArtifactsDAO;
+import com.cognifide.aet.vs.DBKey;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
+import org.apache.camel.Message;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.apache.camel.impl.DefaultExchange;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RemoveArtifactsProcessorTest {
+
+  private static final Set<String> ONE_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays
+      .asList("1", "2", "3", "4", "5", "6", "7"));
+
+  private static final Set<String> ONE_TO_FIVE_ARTIFACTS_ID_SET = new HashSet<>(Arrays
+      .asList("1", "2", "3", "4", "5"));
+
+  private static final Set<String> SIX_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays
+      .asList("6", "7"));
+
+  private Exchange exchange;
+
+  @Mock
+  private ArtifactsDAO artifactDAO;
+
+  @Mock
+  private CleanerContext cleanerContext;
+
+  @Mock
+  private DBKey dbKey;
+
+  private RemoveArtifactsProcessor removeArtifactsProcessor;
+
+  @Before
+  public void setUp() {
+    when(cleanerContext.isDryRun()).thenReturn(false);
+
+    CamelContext ctx = new DefaultCamelContext();
+    exchange = new DefaultExchange(ctx);
+
+    Message in = exchange.getIn();
+    in.setBody(new ReferencedArtifactsMessageBody("", dbKey));
+    in.setHeader(CleanerContext.KEY_NAME, cleanerContext);
+  }
+
+  @Test
+  public void checkIfRemoveArtifactsWasCalled_whenDryRunIsFalse_expectTrue() throws Exception {
+    when(artifactDAO.getArtifactsIds(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
+
+    removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO);
+    removeArtifactsProcessor.process(exchange);
+
+    verify(artifactDAO, times(1)).removeArtifacts(any(DBKey.class), any(Set.class));
+    verify(artifactDAO, times(1)).getArtifactsIds(any(DBKey.class));
+  }
+
+  @Test
+  public void checkIfRemoveArtifactsWasCalled_whenDryRunIsTrue_expectFalse() throws Exception {
+    when(cleanerContext.isDryRun()).thenReturn(true);
+    when(artifactDAO.getArtifactsIds(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
+
+    removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO);
+    removeArtifactsProcessor.process(exchange);
+
+    verify(artifactDAO, times(0)).removeArtifacts(any(DBKey.class), any(Set.class));
+    verify(artifactDAO, times(1)).getArtifactsIds(any(DBKey.class));
+  }
+
+  @Test
+  public void check_substractArtifactsSets_expectSetOfTwoVariables() throws Exception {
+    when(artifactDAO.getArtifactsIds(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    assertEquals(SIX_TO_SEVEN_ARTIFACTS_ID_SET,
+        removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
+  }
+
+  @Test
+  public void check_substractArtifactsSets_expectEmptySet() throws Exception {
+    when(artifactDAO.getArtifactsIds(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_FIVE_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_SEVEN_ARTIFACTS_ID_SET);
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    assertTrue(removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody).isEmpty());
+  }
+
+  @Test
+  public void check_substractArtifactsSets_expectSetOfFiveVariables() throws Exception {
+    when(artifactDAO.getArtifactsIds(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET);
+
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+
+    assertEquals(ONE_TO_FIVE_ARTIFACTS_ID_SET,
+        removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
+  }
+
+  @Test
+  public void check_substractArtifactsSetsWhenDbIsEmpty_expectEmptySet() throws Exception {
+    when(artifactDAO.getArtifactsIds(any(DBKey.class)))
+        .thenReturn(new HashSet<>());
+    setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET);
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    assertTrue(removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody).isEmpty());
+  }
+
+  private void setArtifactsIdToKeep(Set<String> artifactsIdToKeep) {
+    ReferencedArtifactsMessageBody body = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    body.setArtifactsToKeep(artifactsIdToKeep);
+  }
+
+}

core/datastorage/src/main/java/com/cognifide/aet/vs/artifacts/ArtifactsDAOMongoDBImpl.java

@@ -15,6 +15,8 @@
  */
 package com.cognifide.aet.vs.artifacts;
 
+import static com.mongodb.client.model.Projections.fields;
+import static com.mongodb.client.model.Projections.include;
 import com.cognifide.aet.vs.Artifact;
 import com.cognifide.aet.vs.ArtifactsDAO;
 import com.cognifide.aet.vs.DBKey;
@@ -30,6 +32,7 @@
 import java.lang.reflect.Type;
 import java.nio.charset.StandardCharsets;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.Set;
 import org.apache.commons.io.IOUtils;
 import org.bson.Document;
@@ -103,6 +106,22 @@ public Artifact getArtifact(DBKey dbKey, String objectID) {
     return artifact;
   }
 
+  @Override
+  public Set<String> getArtifactsIds(DBKey dbKey) {
+    final String dbName = MongoDBClient.getDbName(dbKey.getCompany(), dbKey.getProject());
+    FindIterable<Document> findIterable =
+        client.getDatabase(dbName)
+            .getCollection(ARTIFACTS_COLLECTION_NAME + FILES_COLLECTION_SUFFIX)
+            .find().projection(fields(include(ID_FIELD_NAME)));
+
+    Set<String> artifactsId = new HashSet<>();
+    for (Document document : findIterable) {
+      artifactsId.add(document.get(ID_FIELD_NAME).toString());
+    }
+    return artifactsId;
+  }
+
+
   @Override
   public String getArtifactAsString(DBKey dbKey, String objectID) throws IOException {
     String result = null;