Add Ordering, Orderable and @OrderWith.

These APIs allow arbitrary ordernig of tests, including randomization.

src/main/java/junit/framework/JUnit4TestAdapter.java

@@ -9,11 +9,13 @@
 import org.junit.runner.Runner;
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.manipulation.Filterable;
+import org.junit.runner.manipulation.GenericOrdering;
+import org.junit.runner.manipulation.InvalidOrderingException;
 import org.junit.runner.manipulation.NoTestsRemainException;
-import org.junit.runner.manipulation.Sortable;
+import org.junit.runner.manipulation.Orderable;
 import org.junit.runner.manipulation.Sorter;
 
-public class JUnit4TestAdapter implements Test, Filterable, Sortable, Describable {
+public class JUnit4TestAdapter implements Test, Filterable, Orderable, Describable {
     private final Class<?> fNewTestClass;
 
     private final Runner fRunner;
@@ -83,4 +85,8 @@ public void filter(Filter filter) throws NoTestsRemainException {
     public void sort(Sorter sorter) {
         sorter.apply(fRunner);
     }
+
+    public void order(GenericOrdering ordering) throws InvalidOrderingException {
+        ordering.apply(fRunner);
+    }
 }

src/main/java/org/junit/internal/runners/JUnit38ClassRunner.java

@@ -1,5 +1,8 @@
 package org.junit.internal.runners;
 
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+
 import junit.extensions.TestDecorator;
 import junit.framework.AssertionFailedError;
 import junit.framework.Test;
@@ -12,15 +15,16 @@
 import org.junit.runner.Runner;
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.manipulation.Filterable;
+import org.junit.runner.manipulation.GenericOrdering;
+import org.junit.runner.manipulation.InvalidOrderingException;
 import org.junit.runner.manipulation.NoTestsRemainException;
+import org.junit.runner.manipulation.Orderable;
 import org.junit.runner.manipulation.Sortable;
 import org.junit.runner.manipulation.Sorter;
 import org.junit.runner.notification.Failure;
 import org.junit.runner.notification.RunNotifier;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Method;
 
-public class JUnit38ClassRunner extends Runner implements Filterable, Sortable {
+public class JUnit38ClassRunner extends Runner implements Filterable, Orderable {
     private static final class OldTestClassAdaptingListener implements
             TestListener {
         private final RunNotifier notifier;
@@ -170,6 +174,13 @@ public void sort(Sorter sorter) {
         }
     }
 
+    public void order(GenericOrdering ordering) throws InvalidOrderingException {
+        if (getTest() instanceof Orderable) {
+            Orderable adapter = (Orderable) getTest();
+            adapter.order(ordering);
+        }
+    }
+
     private void setTest(Test test) {
         this.test = test;
     }

src/main/java/org/junit/runner/OrderWith.java

@@ -0,0 +1,26 @@
+package org.junit.runner;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.runner.manipulation.Ordering;
+
+/**
+ * When a test class is annotated with <code>&#064;OrderWith</code> or extends a class annotated
+ * with <code>&#064;OrderWith</code>, JUnit will order the tests in the test class (and child
+ * test classes, if any) using the ordering defined by the {@link Ordering} class.
+ *
+ * @since 4.13
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+@Inherited
+public @interface OrderWith {
+    /**
+     * @return a class that extends {@link Ordering} (must have a public no-argument constructor)
+     */
+    Class<? extends Ordering> value();
+}

src/main/java/org/junit/runner/Request.java

@@ -8,6 +8,8 @@
 import org.junit.internal.requests.SortingRequest;
 import org.junit.internal.runners.ErrorReportingRunner;
 import org.junit.runner.manipulation.Filter;
+import org.junit.runner.manipulation.InvalidOrderingException;
+import org.junit.runner.manipulation.Ordering;
 import org.junit.runners.model.InitializationError;
 
 /**
@@ -148,15 +150,15 @@ public Request filterWith(final Description desiredDescription) {
      * For example, here is code to run a test suite in alphabetical order:
      * <pre>
      * private static Comparator&lt;Description&gt; forward() {
-     * return new Comparator&lt;Description&gt;() {
-     * public int compare(Description o1, Description o2) {
-     * return o1.getDisplayName().compareTo(o2.getDisplayName());
-     * }
-     * };
+     *   return new Comparator&lt;Description&gt;() {
+     *     public int compare(Description o1, Description o2) {
+     *       return o1.getDisplayName().compareTo(o2.getDisplayName());
+     *     }
+     *   };
      * }
      *
      * public static main() {
-     * new JUnitCore().run(Request.aClass(AllTests.class).sortWith(forward()));
+     *   new JUnitCore().run(Request.aClass(AllTests.class).sortWith(forward()));
      * }
      * </pre>
      *
@@ -166,4 +168,44 @@ public Request filterWith(final Description desiredDescription) {
     public Request sortWith(Comparator<Description> comparator) {
         return new SortingRequest(this, comparator);
     }
+
+    /**
+     * Returns a Request whose Tests can be run in a certain order, defined by
+     * <code>ordering</code>
+     * <p>
+     * For example, here is code to run a test suite in reverse order:
+     * <pre>
+     * private static Ordering reverse() {
+     *   return new Ordering() {
+     *     public List&lt;Description&gt; order(Collection&lt;Description&gt; siblings) {
+     *       List&lt;Description&gt; ordered = new ArrayList&lt;&gt;(siblings);
+     *       Collections.reverse(ordered);
+     *       return ordered;
+     *     }
+     *   }
+     * }
+     *     
+     * public static main() {
+     *   new JUnitCore().run(Request.aClass(AllTests.class).orderWith(reverse()));
+     * }
+     * </pre>
+     *
+     * @return a Request with ordered Tests
+     * @since 4.13
+     */
+    public Request orderWith(final Ordering ordering) {
+        final Request delegate = this;
+        return new Request() {
+            @Override
+            public Runner getRunner() {
+                try {
+                    Runner runner = delegate.getRunner();
+                    ordering.apply(runner);
+                    return runner;
+                } catch (InvalidOrderingException e) {
+                    return new ErrorReportingRunner(ordering.getClass(), e);
+                }
+            }
+        };
+    }
 }

src/main/java/org/junit/runner/manipulation/GenericOrdering.java

@@ -0,0 +1,32 @@
+package org.junit.runner.manipulation;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.runner.Description;
+
+/**
+ * An {@link Ordering} that is not a {@link Sorter}.
+ *
+ * @since 4.13
+ */
+public final class GenericOrdering extends Ordering {
+    private final Ordering delegate;
+
+    GenericOrdering(Ordering delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public List<Description> order(Collection<Description> siblings) {
+        return delegate.order(siblings);
+    }
+
+    @Override
+    public void apply(Object runner) throws InvalidOrderingException {
+        if (runner instanceof Orderable) {
+            Orderable orderable = (Orderable) runner;
+            orderable.order(this);
+        }
+    }
+}

src/main/java/org/junit/runner/manipulation/InvalidOrderingException.java

@@ -0,0 +1,21 @@
+package org.junit.runner.manipulation;
+
+/**
+ * Thrown when an ordering does something invalid (like remove or add children)
+ *
+ * @since 4.13
+ */
+public class InvalidOrderingException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public InvalidOrderingException() {
+    }
+
+    public InvalidOrderingException(String message) {
+        super(message);
+    }
+
+    public InvalidOrderingException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/org/junit/runner/manipulation/Orderable.java

@@ -0,0 +1,20 @@
+package org.junit.runner.manipulation;
+
+/**
+ * Interface for runners that allow ordering of tests.
+ *
+ * <p>Beware of using this interface to cope with order dependencies between tests.
+ * Tests that are isolated from each other are less expensive to maintain and
+ * can be run individually.
+ *
+ * @since 4.13
+ */
+public interface Orderable extends Sortable {
+
+    /**
+     * Orders the tests using <code>ordering</code>
+     *
+     * @throws InvalidOrderingException if ordering does something invalid (like remove or add children)
+     */
+    void order(GenericOrdering ordering) throws InvalidOrderingException;
+}

src/main/java/org/junit/runner/manipulation/Ordering.java

@@ -0,0 +1,71 @@
+package org.junit.runner.manipulation;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.runner.Description;
+
+/**
+ * Reorders tests. An {@code Ordering} can reverse the order of tests, sort the
+ * order or even shuffle the order.
+ * 
+ * <p>In general you will not need to use a <code>Ordering</code> directly.
+ * Instead, use {@link org.junit.runner.Request#orderWith(Ordering)}.
+ *
+ * @since 4.13
+ */
+public abstract class Ordering {
+
+    /**
+     * Creates an {@link Ordering} that shuffles the items using the given
+     * {@link Random} instance.
+     */
+    public static Ordering shuffledBy(final Random random) {
+        return new Ordering() {
+            @Override
+            public List<Description> order(Collection<Description> siblings) {
+                List<Description> shuffled = new ArrayList<Description>(siblings);
+                Collections.shuffle(shuffled, random);
+                return shuffled;
+            }
+        };
+    }
+
+    /**
+     * Creates an {@link Ordering} from the given class. The class must have a public no-argument constructor.
+     *
+     * @throws InvalidOrderingException if the instance could not be created
+     */
+    public static Ordering definedBy(Class<? extends Ordering> orderingClass)
+            throws InvalidOrderingException {
+        try {
+            return orderingClass.newInstance();
+        } catch (InstantiationException e) {
+            throw new InvalidOrderingException("Could not create ordering", e);
+        } catch (IllegalAccessException e) {
+            throw new InvalidOrderingException("Could not create ordering", e);
+        }
+    }
+
+    /**
+     * Order the tests in <code>runner</code> using this ordering.
+     *
+     * @throws InvalidOrderingException if ordering does something invalid (like remove or add children)
+     */
+    public void apply(Object runner) throws InvalidOrderingException {
+        if (runner instanceof Orderable) {
+            Orderable orderable = (Orderable) runner;
+            orderable.order(new GenericOrdering(this));
+        }
+    }
+
+    /**
+     * Orders the given descriptions (all of which have the same parent).
+     *
+     * @param siblings unmodifiable collection of descriptions to order
+     */
+    public abstract List<Description> order(Collection<Description> siblings);
+}

src/main/java/org/junit/runner/manipulation/Sortable.java

@@ -15,6 +15,6 @@ public interface Sortable {
      *
      * @param sorter the {@link Sorter} to use for sorting the tests
      */
-    public void sort(Sorter sorter);
+    void sort(Sorter sorter);
 
 }

src/main/java/org/junit/runner/manipulation/Sorter.java

@@ -1,6 +1,10 @@
 package org.junit.runner.manipulation;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
+import java.util.List;
 
 import org.junit.runner.Description;
 
@@ -10,7 +14,7 @@
  *
  * @since 4.0
  */
-public class Sorter implements Comparator<Description> {
+public class Sorter extends Ordering implements Comparator<Description> {
     /**
      * NULL is a <code>Sorter</code> that leaves elements in an undefined order
      */
@@ -33,16 +37,32 @@ public Sorter(Comparator<Description> comparator) {
     }
 
     /**
-     * Sorts the test in <code>runner</code> using <code>comparator</code>
+     * Sorts the test in <code>runner</code> using <code>comparator</code>.
      */
-    public void apply(Object object) {
-        if (object instanceof Sortable) {
-            Sortable sortable = (Sortable) object;
+    @Override
+    public void apply(Object runner) {
+        // Sorting is more efficient than ordering, so check if the runner is Sortable first
+        if (runner instanceof Sortable) {
+            Sortable sortable = (Sortable) runner;
             sortable.sort(this);
+        } else {
+            try {
+                super.apply(runner);
+            } catch (InvalidOrderingException e) {
+                // Can never get here when applying a sortable ordering.
+                throw new AssertionError(e);
+            }
         }
     }
 
     public int compare(Description o1, Description o2) {
         return comparator.compare(o1, o2);
     }
+
+    @Override
+    public final List<Description> order(Collection<Description> siblings) {
+        List<Description> sorted = new ArrayList<Description>(siblings);
+        Collections.sort(sorted, this);
+        return sorted;
+    }
 }