Merge #33233

src/bin/sage-runtests

@@ -99,7 +99,9 @@ if __name__ == "__main__":
         help="print a summary at the end of each file of optional tests that were skipped")
 
     parser.add_argument("--stats_path", "--stats-path", default=os.path.join(DOT_SAGE, "timings2.json"),
-                        help="path to a json dictionary for the latest run storing a timing for each file")
+                        help="path to a json dictionary for timings and failure status for each file from previous runs; it will be updated in this run")
+    parser.add_argument("--baseline_stats_path", "--baseline-stats-path", default=None,
+                        help="path to a json dictionary for timings and failure status for each file, to be used as a baseline; it will not be updated")
 
     class GCAction(argparse.Action):
         def __call__(self, parser, namespace, values, option_string=None):

src/sage/doctest/control.py

@@ -124,6 +124,7 @@ def __init__(self, **kwds):
         self.new = False
         self.show_skipped = False
         self.target_walltime = -1
+        self.baseline_stats_path = None
 
         # sage-runtests contains more optional tags. Technically, adding
         # auto_optional_tags here is redundant, since that is added
@@ -456,6 +457,9 @@ def __init__(self, options, args):
 
         self.stats = {}
         self.load_stats(options.stats_path)
+        self.baseline_stats = {}
+        if options.baseline_stats_path:
+            self.load_baseline_stats(options.baseline_stats_path)
         self._init_warn_long()
 
         if self.options.random_seed is None:
@@ -568,6 +572,45 @@ def load_environment(self):
         from importlib import import_module
         return import_module(self.options.environment)
 
+    def load_baseline_stats(self, filename):
+        """
+        Load baseline stats.
+
+        This must be a JSON file in the same format that :meth:`load_stats`
+        expects.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults, DocTestController
+            sage: DC = DocTestController(DocTestDefaults(), [])
+            sage: import json
+            sage: filename = tmp_filename()
+            sage: with open(filename, 'w') as stats_file:
+            ....:     json.dump({'sage.doctest.control':{'failed':True}}, stats_file)
+            sage: DC.load_baseline_stats(filename)
+            sage: DC.baseline_stats['sage.doctest.control']
+            {'failed': True}
+
+        If the file doesn't exist, nothing happens. If there is an
+        error, print a message. In any case, leave the stats alone::
+
+            sage: d = tmp_dir()
+            sage: DC.load_baseline_stats(os.path.join(d))  # Cannot read a directory
+            Error loading baseline stats from ...
+            sage: DC.load_baseline_stats(os.path.join(d, "no_such_file"))
+            sage: DC.baseline_stats['sage.doctest.control']
+            {'failed': True}
+        """
+        # Simply ignore non-existing files
+        if not os.path.exists(filename):
+            return
+
+        try:
+            with open(filename) as stats_file:
+                self.baseline_stats.update(json.load(stats_file))
+        except Exception:
+            self.log("Error loading baseline stats from %s"%filename)
+
     def load_stats(self, filename):
         """
         Load stats from the most recent run(s).

src/sage/doctest/reporting.py

@@ -378,6 +378,10 @@ def report(self, source, timeout, return_code, results, output, pid=None):
             postscript = self.postscript
             stats = self.stats
             basename = source.basename
+            if self.controller.baseline_stats:
+                the_baseline_stats = self.controller.baseline_stats.get(basename, {})
+            else:
+                the_baseline_stats = {}
             cmd = self.report_head(source)
             try:
                 ntests, result_dict = results
@@ -398,25 +402,31 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                         fail_msg += " (and interrupt failed)"
                     else:
                         fail_msg += " (with %s after interrupt)"%signal_name(sig)
+                if the_baseline_stats.get('failed', False):
+                    failmsg += " [failed in baseline]"
                 log("    %s\n%s\nTests run before %s timed out:"%(fail_msg, "*"*70, process_name))
                 log(output)
                 log("*"*70)
                 postscript['lines'].append(cmd + "  # %s"%fail_msg)
                 stats[basename] = dict(failed=True, walltime=1e6)
-                self.error_status |= 4
+                if not the_baseline_stats.get('failed', False):
+                    self.error_status |= 4
             elif return_code:
                 if return_code > 0:
                     fail_msg = "Bad exit: %s"%return_code
                 else:
                     fail_msg = "Killed due to %s"%signal_name(-return_code)
                 if ntests > 0:
                     fail_msg += " after testing finished"
+                if the_baseline_stats.get('failed', False):
+                    failmsg += " [failed in baseline]"
                 log("    %s\n%s\nTests run before %s failed:"%(fail_msg,"*"*70, process_name))
                 log(output)
                 log("*"*70)
                 postscript['lines'].append(cmd + "  # %s" % fail_msg)
                 stats[basename] = dict(failed=True, walltime=1e6)
-                self.error_status |= (8 if return_code > 0 else 16)
+                if not the_baseline_stats.get('failed', False):
+                    self.error_status |= (8 if return_code > 0 else 16)
             else:
                 if hasattr(result_dict, 'walltime') and hasattr(result_dict.walltime, '__len__') and len(result_dict.walltime) > 0:
                     wall = sum(result_dict.walltime) / len(result_dict.walltime)
@@ -477,8 +487,12 @@ def report(self, source, timeout, return_code, results, output, pid=None):
                 if result_dict.err is None or result_dict.err == 'tab':
                     f = result_dict.failures
                     if f:
-                        postscript['lines'].append(cmd + "  # %s failed" % (count_noun(f, "doctest")))
-                        self.error_status |= 1
+                        failmsg = "%s failed" % (count_noun(f, "doctest"))
+                        if the_baseline_stats.get('failed', False):
+                            failmsg += " [failed in baseline]"
+                        postscript['lines'].append(cmd + "  # %s" % failmsg)
+                        if not the_baseline_stats.get('failed', False):
+                            self.error_status |= 1
                     if f or result_dict.err == 'tab':
                         stats[basename] = dict(failed=True, walltime=wall)
                     else: