Allow data providers to be non cacheable

Closes testng-team#3041

CHANGES.txt

@@ -1,4 +1,5 @@
 Current (7.10.0)
+Fixed: GITHUB-3041: TestNG 7.x DataProvider works in opposite to TestNG 6.x when retrying tests. (Krishnan Mahadevan)
 Fixed: GITHUB-3066: How to dynamically adjust the number of TestNG threads after IExecutorFactory is deprecated? (Krishnan Mahadevan)
 New:   GITHUB-2874: Allow users to define ordering for TestNG listeners (Krishnan Mahadevan)
 Fixed: GITHUB-3033: Moved ant support under own repository https://github.com/testng-team/testng-ant (Julien Herr)

testng-core-api/src/main/java/org/testng/IDataProviderMethod.java

@@ -38,4 +38,12 @@ default boolean propagateFailureAsTestFailure() {
   default Class<? extends IRetryDataProvider> retryUsing() {
     return IRetryDataProvider.DisableDataProviderRetries.class;
   }
+
+  /**
+   * @return - <code>true</code> if TestNG should use data returned by the original data provider
+   *     invocation, when a test method fails and is configured to be retried.
+   */
+  default boolean cachedDataForTestRetries() {
+    return true;
+  }
 }

testng-core-api/src/main/java/org/testng/annotations/DataProvider.java

@@ -57,6 +57,12 @@
    */
   boolean propagateFailureAsTestFailure() default false;
 
+  /**
+   * @return - <code>true</code> if TestNG should use data returned by the original data provider
+   *     invocation, when a test method fails and is configured to be retried.
+   */
+  boolean cacheDataForTestRetries() default true;
+
   /**
    * @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a
    *     data provider.

testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java

@@ -40,4 +40,16 @@ public interface IDataProviderAnnotation extends IAnnotation {
    *     data provider.
    */
   Class<? extends IRetryDataProvider> retryUsing();
+
+  /**
+   * @param cache - when set to <code>true</code>, TestNG does not invoke the data provider again
+   *     when retrying failed tests using a retry analyzer.
+   */
+  void cacheDataForTestRetries(boolean cache);
+
+  /**
+   * @return - <code>true</code> if TestNG should use data returned by the original data provider
+   *     invocation, when a test method fails and is configured to be retried.
+   */
+  boolean isCacheDataForTestRetries();
 }

testng-core/src/main/java/org/testng/internal/DataProviderMethod.java

@@ -53,4 +53,9 @@ public boolean propagateFailureAsTestFailure() {
   public Class<? extends IRetryDataProvider> retryUsing() {
     return annotation.retryUsing();
   }
+
+  @Override
+  public boolean cachedDataForTestRetries() {
+    return annotation.isCacheDataForTestRetries();
+  }
 }

testng-core/src/main/java/org/testng/internal/Parameters.java

@@ -442,7 +442,7 @@ private static void checkParameterTypes(
         throw new TestNGException(
             errPrefix
                 + ".\nFor more information on native dependency injection please refer to "
-                + "https://testng.org/doc/documentation-main.html#native-dependency-injection");
+                + "https://testng.org/#_dependency_injection");
       }
     }
 

testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java

@@ -13,6 +13,8 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi
   private boolean m_bubbleUpFailures = false;
   private Class<? extends IRetryDataProvider> retryUsing;
 
+  private boolean cachedDataForTestRetries = true;
+
   @Override
   public boolean isParallel() {
     return m_parallel;
@@ -62,4 +64,13 @@ public void setRetryUsing(Class<? extends IRetryDataProvider> retry) {
   public Class<? extends IRetryDataProvider> retryUsing() {
     return retryUsing;
   }
+
+  public void cacheDataForTestRetries(boolean cache) {
+    this.cachedDataForTestRetries = cache;
+  }
+
+  @Override
+  public boolean isCacheDataForTestRetries() {
+    return cachedDataForTestRetries;
+  }
 }

testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java

@@ -483,6 +483,7 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) {
       result.propagateFailureAsTestFailure();
     }
     result.setRetryUsing(c.retryUsing());
+    result.cacheDataForTestRetries(c.cacheDataForTestRetries());
     return result;
   }
 

testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java

@@ -23,6 +23,7 @@
 import org.testng.DataProviderHolder;
 import org.testng.IClassListener;
 import org.testng.IDataProviderListener;
+import org.testng.IDataProviderMethod;
 import org.testng.IHookable;
 import org.testng.IInvokedMethod;
 import org.testng.IInvokedMethodListener;
@@ -238,6 +239,24 @@ public FailureContext retryFailed(
     failure.representsRetriedMethod.set(true);
     do {
       failure.instances = Lists.newArrayList();
+      boolean cacheData =
+          Optional.ofNullable(arguments.getTestMethod().getDataProviderMethod())
+              .map(IDataProviderMethod::cachedDataForTestRetries)
+              .orElse(false);
+      if (!cacheData) {
+        Map<String, String> allParameters = Maps.newHashMap();
+        int verbose = testContext.getCurrentXmlTest().getVerbose();
+        ParameterHandler handler =
+            new ParameterHandler(
+                m_configuration.getObjectFactory(), annotationFinder(), this.holder, verbose);
+
+        ParameterBag bag =
+            handler.createParameters(
+                arguments.getTestMethod(), arguments.getParameters(), allParameters, testContext);
+        if (bag.hasErrors()) {
+          continue;
+        }
+      }
       Object[] parameterValues = arguments.getParameterValues();
       TestMethodArguments tma =
           new TestMethodArguments.Builder()

testng-core/src/test/java/test/dataprovider/DataProviderTest.java

@@ -55,12 +55,21 @@
 import test.dataprovider.issue2934.TestCaseSample.CoreListener;
 import test.dataprovider.issue2934.TestCaseSample.ToggleDataProvider;
 import test.dataprovider.issue2980.LoggingListener;
+import test.dataprovider.issue3041.SampleTestCase;
 import test.dataprovider.issue3045.DataProviderListener;
 import test.dataprovider.issue3045.DataProviderTestClassSample;
 import test.dataprovider.issue3045.DataProviderWithoutListenerTestClassSample;
 
 public class DataProviderTest extends SimpleBaseTest {
 
+  @Test(description = "GITHUB-3041")
+  public void ensureDataProvidersCanBeInstructedNotToCacheDataForFailedTestRetries() {
+    TestNG testng = create(SampleTestCase.class);
+    testng.setVerbose(2);
+    testng.run();
+    assertThat(SampleTestCase.invocationCount.get()).isEqualTo(2);
+  }
+
   @Test(description = "GITHUB-2819")
   public void testDataProviderCanBeRetriedOnFailures() {
     TestNG testng = create(TestClassUsingDataProviderRetrySample.class);

testng-core/src/test/java/test/dataprovider/issue3041/SampleTestCase.java

@@ -0,0 +1,41 @@
+package test.dataprovider.issue3041;
+
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.testng.IRetryAnalyzer;
+import org.testng.ITestResult;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class SampleTestCase {
+
+  public static final AtomicInteger invocationCount = new AtomicInteger(0);
+  private static final Random random = new Random();
+
+  @Test(dataProvider = "dp", retryAnalyzer = MyRetry.class)
+  public void testMethod(int i) {
+    if (invocationCount.get() != 2) {
+      throw new RuntimeException("Failed for " + i);
+    }
+  }
+
+  @DataProvider(name = "dp", cacheDataForTestRetries = false)
+  public Object[][] getData() {
+    invocationCount.incrementAndGet();
+    return new Object[][] {{next()}, {next()}};
+  }
+
+  private static int next() {
+    return random.nextInt();
+  }
+
+  public static class MyRetry implements IRetryAnalyzer {
+
+    private final AtomicInteger counter = new AtomicInteger(1);
+
+    @Override
+    public boolean retry(ITestResult result) {
+      return counter.getAndIncrement() != 2;
+    }
+  }
+}