Add --metric/-m option to eval command to select metric(s)

annif/cli.py

@@ -313,6 +313,8 @@ def run_index(project_id, directory, suffix, force,
 @click.option('--docs-limit', '-d', default=None,
               type=click.IntRange(0, None),
               help='Maximum number of documents to use')
+@click.option('--metric', '-m', default=[], multiple=True,
+              help='Metric to calculate (default: all)')
 @click.option(
     '--metrics-file',
     '-M',
@@ -345,6 +347,7 @@ def run_eval(
         limit,
         threshold,
         docs_limit,
+        metric,
         metrics_file,
         results_file,
         jobs,
@@ -386,12 +389,12 @@ def run_eval(
                                 annif.corpus.SubjectSet((uris, labels)))
 
     template = "{0:<30}\t{1}"
-    metrics = eval_batch.results(results_file=results_file)
+    metrics = eval_batch.results(metrics=metric, results_file=results_file)
     for metric, score in metrics.items():
         click.echo(template.format(metric + ":", score))
     if metrics_file:
         json.dump(
-            {metric_code(metric): val for metric, val in metrics.items()},
+            {metric_code(mname): val for mname, val in metrics.items()},
             metrics_file, indent=2)
 
 

annif/eval.py

@@ -96,7 +96,7 @@ def __init__(self, subject_index):
     def evaluate(self, hits, gold_subjects):
         self._samples.append((hits, gold_subjects))
 
-    def _evaluate_samples(self, y_true, y_pred, metrics='all'):
+    def _evaluate_samples(self, y_true, y_pred, metrics=[]):
         y_pred_binary = y_pred > 0.0
         y_true_sparse = csr_matrix(y_true)
 
@@ -150,7 +150,7 @@ def _evaluate_samples(self, y_true, y_pred, metrics='all'):
                 y_true, y_pred_binary),
         }
 
-        if metrics == 'all':
+        if not metrics:
             metrics = all_metrics.keys()
 
         with warnings.catch_warnings():
@@ -202,9 +202,9 @@ def output_result_per_subject(self, y_true, y_pred, results_file):
         self._result_per_subject_header(results_file)
         self._result_per_subject_body(zipped, results_file)
 
-    def results(self, metrics='all', results_file=None, warnings=False):
+    def results(self, metrics=[], results_file=None, warnings=False):
         """evaluate a set of selected subjects against a gold standard using
-        different metrics. The set of metrics can be either 'all' or 'simple'.
+        different metrics. If metrics is empty, use all available metrics.
         If results_file (file object) given, write results per subject to it"""
 
         if not self._samples:

tests/test_cli.py

@@ -520,6 +520,27 @@ def test_eval_param(tmpdir):
     assert float(recall.group(1)) == 0.0
 
 
+def test_eval_metric(tmpdir):
+    tmpdir.join('doc1.txt').write('doc1')
+    tmpdir.join('doc1.key').write('dummy')
+    tmpdir.join('doc2.txt').write('doc2')
+    tmpdir.join('doc2.key').write('none')
+    tmpdir.join('doc3.txt').write('doc3')
+    result = runner.invoke(
+        annif.cli.cli, [
+            'eval', '--metric', 'F1@5', '-m', 'NDCG', 'dummy-en',
+            str(tmpdir)])
+    assert not result.exception
+    assert result.exit_code == 0
+
+    f1 = re.search(r'F1@5\s*:\s+(\d.\d+)', result.output)
+    assert float(f1.group(1)) > 0.0
+    ndcg = re.search(r'NDCG\s*:\s+(\d.\d+)', result.output)
+    assert float(ndcg.group(1)) > 0.0
+    # check that we only have 2 metrics + "Documents evaluated"
+    assert len(result.output.strip().split('\n')) == 3
+
+
 def test_eval_metricsfile(tmpdir):
     tmpdir.join('doc1.txt').write('doc1')
     tmpdir.join('doc1.key').write('dummy')