Add config key for callback discrepancy behavior

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB2818: Add configuration key for callback discrepancy behavior (Krishnan Mahadevan)
 Fixed: GITHUB-2819: Ability to retry a data provider in case of failures (Krishnan Mahadevan)
 Fixed: GITHUB-2308: StringIndexOutOfBoundsException in findClassesInPackage - Surefire/Maven - JDK 11 fails (Krishnan Mahadevan)
 Fixed: GITHUB:2788: TestResult.isSuccess() is TRUE when test fails due to expectedExceptions (Krishnan Mahadevan)

testng-core-api/src/main/java/org/testng/internal/RuntimeBehavior.java

@@ -14,9 +14,14 @@ public final class RuntimeBehavior {
   private static final String MEMORY_FRIENDLY_MODE = "testng.memory.friendly";
   public static final String STRICTLY_HONOUR_PARALLEL_MODE = "testng.strict.parallel";
   public static final String TESTNG_DEFAULT_VERBOSE = "testng.default.verbose";
+  public static final String IGNORE_CALLBACK_INVOCATION_SKIPS = "testng.ignore.callback.skip";
 
   private RuntimeBehavior() {}
 
+  public static boolean ignoreCallbackInvocationSkips() {
+    return Boolean.getBoolean(IGNORE_CALLBACK_INVOCATION_SKIPS);
+  }
+
   public static boolean strictParallelism() {
     return Boolean.getBoolean(STRICTLY_HONOUR_PARALLEL_MODE);
   }

testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java

@@ -362,7 +362,11 @@ private void invokeConfigurationMethod(
             tm, method, targetInstance, params, testResult);
       }
       boolean testStatusRemainedUnchanged = testResult.isNotRunning();
-      if (usesConfigurableInstance && willfullyIgnored && testStatusRemainedUnchanged) {
+      boolean throwException = !RuntimeBehavior.ignoreCallbackInvocationSkips();
+      if (throwException
+          && usesConfigurableInstance
+          && willfullyIgnored
+          && testStatusRemainedUnchanged) {
         throw new ConfigurationNotInvokedException(tm);
       }
       testResult.setStatus(ITestResult.SUCCESS);

testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java

@@ -691,7 +691,11 @@ private ITestResult invokeMethod(
                 hookableInstance);
       }
       boolean testStatusRemainedUnchanged = testResult.isNotRunning();
-      if (usesHookableInstance && willfullyIgnored && testStatusRemainedUnchanged) {
+      boolean throwException = !RuntimeBehavior.ignoreCallbackInvocationSkips();
+      if (throwException
+          && usesHookableInstance
+          && willfullyIgnored
+          && testStatusRemainedUnchanged) {
         TestNotInvokedException tn = new TestNotInvokedException(arguments.tm);
         testResult.setThrowable(tn);
         setTestStatus(testResult, ITestResult.FAILURE);

testng-core/src/test/java/test/hook/HookableTest.java

@@ -16,7 +16,9 @@
 import org.testng.TestNG;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+import org.testng.internal.RuntimeBehavior;
 import test.SimpleBaseTest;
+import test.hook.samples.CallBackSample;
 import test.hook.samples.ConfigurableFailureSample;
 import test.hook.samples.ConfigurableFailureWithStatusAlteredSample;
 import test.hook.samples.ConfigurableSuccessSample;
@@ -196,6 +198,28 @@ public void configurableFailureWithStatusAltered() {
     assertions.assertAll();
   }
 
+  @Test(description = "GITHUB-2818", dataProvider = "callback")
+  public void tesFailuresFromCallbackInvocationSkipsCanBeDisabledViaJVMArgs(
+      Class<?> cls, int expected) {
+    try {
+      System.setProperty(RuntimeBehavior.IGNORE_CALLBACK_INVOCATION_SKIPS, "true");
+      TestNG testng = create(cls);
+      testng.run();
+      int status = testng.getStatus();
+      assertThat(status).isEqualTo(expected);
+    } finally {
+      System.setProperty(RuntimeBehavior.IGNORE_CALLBACK_INVOCATION_SKIPS, "");
+    }
+  }
+
+  @DataProvider(name = "callback")
+  public Object[][] getCallBackClasses() {
+    return new Object[][] {
+      {CallBackSample.ConfigCallBackSkipTestCase.class, 0},
+      {CallBackSample.TestCallBackSkipTestCase.class, 1},
+    };
+  }
+
   public static class TestResultsCollector implements IInvokedMethodListener {
     private final List<ITestResult> passed = new ArrayList<>();
     private final List<ITestResult> invoked = new ArrayList<>();

testng-core/src/test/java/test/hook/samples/CallBackSample.java

@@ -0,0 +1,32 @@
+package test.hook.samples;
+
+import org.testng.IConfigurable;
+import org.testng.IConfigureCallBack;
+import org.testng.IHookCallBack;
+import org.testng.IHookable;
+import org.testng.ITestResult;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class CallBackSample {
+
+  public static class ConfigCallBackSkipTestCase implements IConfigurable {
+
+    @Override
+    public void run(IConfigureCallBack callBack, ITestResult testResult) {}
+
+    @BeforeMethod
+    public void beforeMethod() {}
+
+    @Test
+    public void testMethod() {}
+  }
+
+  public static class TestCallBackSkipTestCase implements IHookable {
+    @Override
+    public void run(IHookCallBack callBack, ITestResult testResult) {}
+
+    @Test
+    public void testMethod() {}
+  }
+}