Add simple coverage metric between two lists

README.md

@@ -37,7 +37,7 @@ julia> Pkg.test("Recommendation")
 Note that unit tests for dataset loaders (e.g., `load_movielens_latest()`) are conditionally triggered as follows, so that CI does not make excessive download requests to the external sites:
 
 ```julia
-julia> Pkg.test("Recommendation", test_args=["download"])
+julia> Pkg.test("Recommendation", test_args=["data", "download"])
 ```
 
 Build documentation contents:

src/base_recommender.jl

@@ -28,7 +28,7 @@ function fit!(recommender::Recommender; kwargs...)
     error("fit! is not implemented for recommender type $(typeof(recommender))")
 end
 
-function recommend(recommender::Recommender, u::Integer, k::Integer, candidates::Array{T}) where {T<:Integer}
+function recommend(recommender::Recommender, u::Integer, k::Integer, candidates::AbstractVector{T}) where {T<:Integer}
     d = Dict{T,AbstractFloat}()
     for candidate in candidates
         score = predict(recommender, u, candidate)

src/metrics/base.jl

@@ -1,5 +1,5 @@
 export Metric, AccuracyMetric, RankingMetric
-export measure, count_true_positive
+export measure, count_intersect, coverage
 
 abstract type Metric end
 
@@ -11,10 +11,14 @@ end
 
 abstract type RankingMetric <: Metric end
 
-function measure(metric::RankingMetric, truth::Array{T}, pred::Array{T}, k::Integer) where T
+function measure(metric::RankingMetric, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer) where T
     error("measure is not implemented for metric type $(typeof(metric))")
 end
 
-function count_true_positive(truth::Array{T}, pred::Array{T}) where T
-    sum(in(truth), pred)
+function count_intersect(s1::Union{AbstractSet, AbstractVector}, s2::Union{AbstractSet, AbstractVector})
+    length(intersect(s1, s2))
+end
+
+function coverage(items::Union{AbstractSet, AbstractVector}, catalog::Union{AbstractSet, AbstractVector})
+    count_intersect(items, catalog) / length(catalog)
 end

src/metrics/ranking.jl

@@ -11,14 +11,14 @@ Here, ``|\\mathcal{I}^+_u \\cap I_N(u)|`` is the number of *true positives*.
 
     measure(
         metric::Recall,
-        truth::Array{T},
-        pred::Array{T},
+        truth::AbstractVector{T},
+        pred::AbstractVector{T},
         k::Integer
     )
 """
 struct Recall <: RankingMetric end
-function measure(metric::Recall, truth::Array{T}, pred::Array{T}, k::Integer) where T
-    count_true_positive(truth, pred[1:k]) / length(truth)
+function measure(metric::Recall, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer) where T
+    count_intersect(truth, pred[1:k]) / length(truth)
 end
 
 """
@@ -31,14 +31,14 @@ Precision-at-``N`` (Precision@``N``) evaluates correctness of a top-``N`` recomm
 
     measure(
         metric::Precision,
-        truth::Array{T},
-        pred::Array{T},
+        truth::AbstractVector{T},
+        pred::AbstractVector{T},
         k::Integer
     )
 """
 struct Precision <: RankingMetric end
-function measure(metric::Precision, truth::Array{T}, pred::Array{T}, k::Integer) where T
-    count_true_positive(truth, pred[1:k]) / k
+function measure(metric::Precision, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer) where T
+    count_intersect(truth, pred[1:k]) / k
 end
 
 """
@@ -53,12 +53,12 @@ It should be noticed that, MAP is not a simple mean of sum of Precision@``1``, P
 
     measure(
         metric::MAP,
-        truth::Array{T},
-        pred::Array{T}
+        truth::AbstractVector{T},
+        pred::AbstractVector{T}
     )
 """
 struct MAP <: RankingMetric end
-function measure(metric::MAP, truth::Array{T}, pred::Array{T}, k::Integer=0) where T
+function measure(metric::MAP, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer=0) where T
     tp = accum = 0
     n_pred = length(pred)
 
@@ -83,12 +83,12 @@ AUC calculation keeps track the number of true positives at different rank in ``
 
     measure(
         metric::AUC,
-        truth::Array{T},
-        pred::Array{T}
+        truth::AbstractVector{T},
+        pred::AbstractVector{T}
     )
 """
 struct AUC <: RankingMetric end
-function measure(metric::AUC, truth::Array{T}, pred::Array{T}, k::Integer=0) where T
+function measure(metric::AUC, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer=0) where T
     tp = correct = 0
     for r in pred
         if findfirst(isequal(r), truth) != nothing
@@ -115,12 +115,12 @@ RR can be zero if and only if ``\\mathcal{I}^+_u`` is empty.
 
     measure(
         metric::ReciprocalRank,
-        truth::Array{T},
-        pred::Array{T}
+        truth::AbstractVector{T},
+        pred::AbstractVector{T}
     )
 """
 struct ReciprocalRank <: RankingMetric end
-function measure(metric::ReciprocalRank, truth::Array{T}, pred::Array{T}, k::Integer=0) where T
+function measure(metric::ReciprocalRank, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer=0) where T
     n_pred = length(pred)
     for n = 1:n_pred
         if findfirst(isequal(pred[n]), truth) != nothing
@@ -143,12 +143,12 @@ MPR internally considers not only top-``N`` recommended items also all of the no
 
     measure(
         metric::MPR,
-        truth::Array{T},
-        pred::Array{T}
+        truth::AbstractVector{T},
+        pred::AbstractVector{T}
     )
 """
 struct MPR <: RankingMetric end
-function measure(metric::MPR, truth::Array{T}, pred::Array{T}, k::Integer=0) where T
+function measure(metric::MPR, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer=0) where T
     accum = 0
     n_pred = length(pred)
     for t in truth
@@ -165,13 +165,13 @@ Like MPR, normalized discounted cumulative gain (NDCG) computes a score for ``I(
 
     measure(
         metric::NDCG,
-        truth::Array{T},
-        pred::Array{T},
+        truth::AbstractVector{T},
+        pred::AbstractVector{T},
         k::Integer
     )
 """
 struct NDCG <: RankingMetric end
-function measure(metric::NDCG, truth::Array{T}, pred::Array{T}, k::Integer) where T
+function measure(metric::NDCG, truth::AbstractVector{T}, pred::AbstractVector{T}, k::Integer) where T
     dcg = idcg = 0
     for n = 1:k
         d = 1 / log2(n + 1)

test/metrics/test_base.jl

@@ -0,0 +1,10 @@
+
+println("-- Testing base module for metrics calculation")
+
+@test count_intersect([1, 2, 3], [1, 2, 3, 4]) == 3
+@test count_intersect(Array{Int, 1}(), [1, 2, 3, 4]) == 0
+@test count_intersect([1, 2, 3], Array{Int, 1}()) == 0
+@test count_intersect(["a", "b", "c"], ["a", "b", "c"]) == 3
+
+@test coverage([1, 2, 3], [1, 2, 3, 4]) == 0.75
+@test coverage(Set(["a", "b", "c", "d"]), Set(["c", "d", "e", "f"])) == 0.5

test/runtests.jl

@@ -4,10 +4,21 @@ using SparseArrays
 
 include("test_utils.jl")
 
-# run testing modules if and only if `name` is in `test_args``
-# e.g. `Pkg.test("Recommendation", test_args=["misc"])`
+# run testing modules when `test_args` is empty or `name` is in the args
+# e.g. `Pkg.test("Recommendation")` => run all test cases under `@testset_default_or_if`
+#      `Pkg.test("Recommendation", test_args=["misc"])` => run a test set `@testset_default_or_if "misc"` only
 # reference: https://github.com/JuliaAI/MLJBase.jl/blob/4a8f3f323f91ee6b6f5fb2b3268729b3101c003c/test/runtests.jl#L52-L62
-macro conditional_testset(name, expr)
+RUN_ALL_TESTS = isempty(ARGS)
+macro testset_default_or_if(name, expr)
+    name = string(name)
+    esc(quote
+        if RUN_ALL_TESTS || $name in ARGS
+            @testset $name $expr
+        end
+    end)
+end
+# run testing modules if and only if `name` is explicitly specified in `test_args`
+macro testset_if(name, expr)
     name = string(name)
     esc(quote
         if $name in ARGS
@@ -16,19 +27,19 @@ macro conditional_testset(name, expr)
     end)
 end
 
-@testset "recommender" begin
+@testset_default_or_if "recommender" begin
     include("test_base_recommender.jl")
     include("test_compat.jl")
 
-    @testset "baseline" begin
+    @testset_default_or_if "baseline" begin
         include("baseline/test_user_mean.jl")
         include("baseline/test_item_mean.jl")
         include("baseline/test_most_popular.jl")
         include("baseline/test_threshold_percentage.jl")
         include("baseline/test_co_occurrence.jl")
     end
 
-    @testset "model" begin
+    @testset_default_or_if "model" begin
         include("model/test_tf_idf.jl")
         include("model/test_user_knn.jl")
         include("model/test_item_knn.jl")
@@ -38,15 +49,16 @@ end
     end
 end
 
-@testset "evaluation" begin
+@testset_default_or_if "evaluation" begin
+    include("metrics/test_base.jl")
     include("metrics/test_accuracy.jl")
     include("metrics/test_ranking.jl")
 
     include("evaluation/test_evaluate.jl")
     include("evaluation/test_cross_validation.jl")
 end
 
-@testset "data" begin
+@testset_default_or_if "data" begin
     include("test_data_accessor.jl")
     include("test_datasets.jl")
     include("test_synthetic.jl")

test/test_datasets.jl

@@ -119,7 +119,7 @@ test_get_data_home()
 test_unzip()
 test_load_libsvm_file()
 
-@conditional_testset "download" begin
+@testset_if "download" begin
     test_download_file()
     test_load_movielens_100k()
     test_load_movielens_latest()