Support running partial test names from suite file

Closes #2897 1. Add new boolean option '-ignoreMissedTestNames' to work with the option '-testnames'. 2. When -testnames is given, and '-ignoreMissedTestNames true' is also given, then in case any missed test names not found in the suite, only warning message will be printed, TestNG will continue to run other test names which are existing in the suite. 3. Users who are going to use the new option '-ignoreMissedTestNames' should be aware of that the logging level should be properly configured to make sure the warning message is visible in output or console, rather than missed the notification of the missed test names, if any.
testng-team · May 6, 2023 · 4c7d903 · 4c7d903
1 parent a766113
commit 4c7d903
Show file tree

Hide file tree

Showing 9 changed files with 304 additions and 33 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,4 +1,5 @@
 Current
+New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen)
 New: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym)
 Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira)
 Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan)

diff --git a/testng-ant/src/main/java/org/testng/TestNGAntTask.java b/testng-ant/src/main/java/org/testng/TestNGAntTask.java
@@ -161,6 +161,7 @@ public class TestNGAntTask extends Task {
   private String m_methods;
   private Mode mode = Mode.testng;
   private boolean forkJvm = true;
+  private boolean m_ignoreMissedTestNames;
 
   public enum Mode {
     // lower-case to better look in build scripts
@@ -360,6 +361,10 @@ public void setTestNames(String testNames) {
     m_testNames = testNames;
   }
 
+  public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+    m_ignoreMissedTestNames = ignoreMissedTestNames;
+  }
+
   /**
    * Sets the suite runner class to invoke
    *
@@ -578,6 +583,7 @@ protected List<String> createArguments() {
     addStringIfNotBlank(argv, CommandLineArgs.SUITE_NAME, m_suiteName);
     addStringIfNotBlank(argv, CommandLineArgs.TEST_NAME, m_testName);
     addStringIfNotBlank(argv, CommandLineArgs.TEST_NAMES, m_testNames);
+    addBooleanIfTrue(argv, CommandLineArgs.IGNORE_MISSED_TEST_NAMES, m_ignoreMissedTestNames);
     addStringIfNotBlank(argv, CommandLineArgs.METHODS, m_methods);
     addReporterConfigs(argv);
     addIntegerIfNotNull(argv, CommandLineArgs.SUITE_THREAD_POOL_SIZE, m_suiteThreadPoolSize);

diff --git a/testng-collections/src/main/java/org/testng/util/Strings.java b/testng-collections/src/main/java/org/testng/util/Strings.java
@@ -1,5 +1,6 @@
 package org.testng.util;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -18,6 +19,23 @@ public static boolean isNotNullAndNotEmpty(String string) {
     return !isNullOrEmpty(string);
   }
 
+  /**
+   * Check if the given string list is null or empty or all elements are null or empty or blank.
+   *
+   * @param list A list instance with String elements.
+   * @return true if the given string list is null or empty or all elements are null or empty or
+   *     blank; otherwise false.
+   */
+  public static boolean isBlankStringList(List<String> list) {
+    if (list == null) {
+      return true;
+    }
+    if (list.isEmpty()) {
+      return true;
+    }
+    return list.stream().allMatch(t -> t == null || t.isBlank());
+  }
+
   private static final Map<String, String> ESCAPE_HTML_MAP = Maps.newLinkedHashMap();
 
   static {

diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java
@@ -142,6 +142,14 @@ public class CommandLineArgs {
   @Parameter(names = TEST_NAMES, description = "The list of test names to run")
   public String testNames;
 
+  public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames";
+
+  @Parameter(
+      names = IGNORE_MISSED_TEST_NAMES,
+      description =
+          "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.")
+  public boolean ignoreMissedTestNames = false;
+
   public static final String TEST_JAR = "-testjar";
 
   @Parameter(names = TEST_JAR, description = "A jar file containing the tests")

diff --git a/testng-core/src/main/java/org/testng/JarFileUtils.java b/testng-core/src/main/java/org/testng/JarFileUtils.java
@@ -23,6 +23,7 @@
 class JarFileUtils {
   private final IPostProcessor processor;
   private final String xmlPathInJar;
+  private final boolean ignoreMissedTestNames;
   private final List<String> testNames;
   private final List<XmlSuite> suites = Lists.newLinkedList();
   private final XmlSuite.ParallelMode mode;
@@ -36,10 +37,28 @@ class JarFileUtils {
       String xmlPathInJar,
       List<String> testNames,
       XmlSuite.ParallelMode mode) {
+    this(processor, xmlPathInJar, testNames, mode, false);
+  }
+
+  JarFileUtils(
+      IPostProcessor processor,
+      String xmlPathInJar,
+      List<String> testNames,
+      boolean ignoreMissedTestNames) {
+    this(processor, xmlPathInJar, testNames, XmlSuite.ParallelMode.NONE, ignoreMissedTestNames);
+  }
+
+  JarFileUtils(
+      IPostProcessor processor,
+      String xmlPathInJar,
+      List<String> testNames,
+      XmlSuite.ParallelMode mode,
+      boolean ignoreMissedTestNames) {
     this.processor = processor;
     this.xmlPathInJar = xmlPathInJar;
     this.testNames = testNames;
     this.mode = mode == null ? XmlSuite.ParallelMode.NONE : mode;
+    this.ignoreMissedTestNames = ignoreMissedTestNames;
   }
 
   List<XmlSuite> extractSuitesFrom(File jarFile) {
@@ -69,6 +88,7 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
       Enumeration<JarEntry> entries = jf.entries();
       File file = java.nio.file.Files.createTempDirectory("testngXmlPathInJar-").toFile();
       String suitePath = null;
+
       while (entries.hasMoreElements()) {
         JarEntry je = entries.nextElement();
         String jeName = je.getName();
@@ -87,24 +107,34 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
           classes.add(constructClassName(je));
         }
       }
+
       if (Strings.isNullOrEmpty(suitePath)) {
+        Utils.log("TestNG", 1, String.format("Not found '%s' in '%s'.", xmlPathInJar, jarFile));
         return false;
       }
+
       Collection<XmlSuite> parsedSuites = Parser.parse(suitePath, processor);
       delete(file);
+      boolean addedSuite = false;
       for (XmlSuite suite : parsedSuites) {
-        // If test names were specified, only run these test names
-        if (testNames != null) {
-          TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames);
-          testNamesMatcher.validateMissMatchedTestNames();
-          suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
-        } else {
+        if (testNames == null) {
           suites.add(suite);
+          addedSuite = true;
+        } else {
+          TestNamesMatcher testNamesMatcher =
+              new TestNamesMatcher(suite, testNames, ignoreMissedTestNames);
+          boolean validationResult = testNamesMatcher.validateMissMatchedTestNames();
+          if (validationResult) {
+            suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
+            addedSuite = true;
+          } else {
+            Utils.error(String.format("None of '%s' found in '%s'.", testNames, suite));
+          }
         }
-        return true;
       }
+
+      return addedSuite;
     }
-    return false;
   }
 
   private void delete(File f) throws IOException {

diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java
@@ -353,7 +353,8 @@ private Collection<XmlSuite> processCommandLineArgs(Collection<XmlSuite> allSuit
         continue;
       }
       // If test names were specified, only run these test names
-      TestNamesMatcher testNamesMatcher = new TestNamesMatcher(s, m_testNames);
+      TestNamesMatcher testNamesMatcher =
+          new TestNamesMatcher(s, m_testNames, m_ignoreMissedTestNames);
       testNamesMatcher.validateMissMatchedTestNames();
       result.addAll(testNamesMatcher.getSuitesMatchingTestNames());
     }
@@ -417,7 +418,8 @@ public void initializeSuitesAndJarFile() {
     File jarFile = new File(m_jarPath);
 
     JarFileUtils utils =
-        new JarFileUtils(getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode);
+        new JarFileUtils(
+            getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode, m_ignoreMissedTestNames);
 
     Collection<XmlSuite> allSuites = utils.extractSuitesFrom(jarFile);
     allSuites.forEach(this::processParallelModeCommandLineArgs);
@@ -799,6 +801,8 @@ public List<ISuiteListener> getSuiteListeners() {
   /** The list of test names to run from the given suite */
   private List<String> m_testNames;
 
+  private boolean m_ignoreMissedTestNames;
+
   private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT;
 
   private boolean m_randomizeSuites = Boolean.FALSE;
@@ -1475,6 +1479,7 @@ protected void configure(CommandLineArgs cla) {
 
     if (cla.testNames != null) {
       setTestNames(Arrays.asList(cla.testNames.split(",")));
+      setIgnoreMissedTestNames(cla.ignoreMissedTestNames);
     }
 
     // Note: can't use a Boolean field here because we are allowing a boolean
@@ -1574,6 +1579,10 @@ protected void configure(CommandLineArgs cla) {
     alwaysRunListeners(cla.alwaysRunListeners);
   }
 
+  private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+    m_ignoreMissedTestNames = ignoreMissedTestNames;
+  }
+
   public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) {
     m_suiteThreadPoolSize = suiteThreadPoolSize;
   }

diff --git a/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java b/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java
@@ -3,19 +3,35 @@
 import java.util.List;
 import org.testng.TestNGException;
 import org.testng.collections.Lists;
+import org.testng.log4testng.Logger;
+import org.testng.util.Strings;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 
-/** The class to work with "-testnames" */
+/**
+ * The class to work with "-testnames", "-ignoreMissedTestNames", and VM argument
+ * "-Dtestng.ignore.missed.testnames". If both "-ignoreMissedTestNames" and VM argument
+ * "-Dtestng.ignore.missed.testnames" are set, then either of them has "true" value will enable the
+ * feature to ingore partially missed test names and run those existing test names.
+ */
 public final class TestNamesMatcher {
 
+  private static final Logger LOGGER = Logger.getLogger(TestNamesMatcher.class);
+
   private final List<XmlSuite> cloneSuites = Lists.newArrayList();
   private final List<String> matchedTestNames = Lists.newArrayList();
   private final List<XmlTest> matchedTests = Lists.newArrayList();
   private final List<String> testNames;
+  private final boolean ignoreMissedTestNames;
 
   public TestNamesMatcher(XmlSuite xmlSuite, List<String> testNames) {
+    this(xmlSuite, testNames, false);
+  }
+
+  public TestNamesMatcher(
+      XmlSuite xmlSuite, List<String> testNames, boolean ignoreMissedTestNames) {
     this.testNames = testNames;
+    this.ignoreMissedTestNames = ignoreMissedTestNames;
     cloneIfContainsTestsWithNamesMatchingAny(xmlSuite, this.testNames);
   }
 
@@ -26,7 +42,7 @@ public TestNamesMatcher(XmlSuite xmlSuite, List<String> testNames) {
    * @param testNames The list of testnames to iterate through
    */
   private void cloneIfContainsTestsWithNamesMatchingAny(XmlSuite xmlSuite, List<String> testNames) {
-    if (testNames == null || testNames.isEmpty()) {
+    if (Strings.isBlankStringList(testNames)) {
       throw new TestNGException("Please provide a valid list of names to check.");
     }
 
@@ -43,13 +59,38 @@ public List<XmlSuite> getSuitesMatchingTestNames() {
     return cloneSuites;
   }
 
-  public void validateMissMatchedTestNames() {
-    List<String> tmpTestNames = Lists.newArrayList();
-    tmpTestNames.addAll(testNames);
-    tmpTestNames.removeIf(matchedTestNames::contains);
-    if (!tmpTestNames.isEmpty()) {
-      throw new TestNGException("The test(s) <" + tmpTestNames + "> cannot be found in suite.");
+  /**
+   * Do validation for testNames and notify users if any testNames are missed in suite. This method
+   * is also used to decide how to run test suite when test names are given. In legacy logic, if
+   * test names are given and exist in suite, then run them; if any of them do not exist in suite,
+   * then throw exception and exit. After ignoreMissedTestNames is introduced, if
+   * ignoreMissedTestNames is enabled, then any of the given test names exist in suite will be run,
+   * and print warning message to tell those test names do not exist in suite.
+   *
+   * @return boolean if ignoreMissedTestNames disabled, then return true if no missed test names in
+   *     suite, otherwise throw TestNGException; if ignoreMissedTestNames enabled, then return true
+   *     if any test names exist in suite, otehrwise (all given test names are missed) throw
+   *     TestNGException.
+   */
+  public boolean validateMissMatchedTestNames() {
+    final List<String> missedTestNames = getMissedTestNames();
+    if (!missedTestNames.isEmpty()) {
+      final String errMsg = "The test(s) <" + missedTestNames + "> cannot be found in suite.";
+      if (ignoreMissedTestNames && !matchedTestNames.isEmpty()) {
+        LOGGER.warn(errMsg);
+        return true;
+      } else {
+        throw new TestNGException(errMsg);
+      }
     }
+    return missedTestNames.isEmpty() && !matchedTestNames.isEmpty();
+  }
+
+  public List<String> getMissedTestNames() {
+    List<String> missedTestNames = Lists.newArrayList();
+    missedTestNames.addAll(testNames);
+    missedTestNames.removeIf(matchedTestNames::contains);
+    return missedTestNames;
   }
 
   public List<XmlTest> getMatchedTests() {