avoid text encoder constructor call get stack overflow

chengchingwen · Mar 31, 2023 · 3cd210f · 3cd210f
1 parent 0d4a5fd
commit 3cd210f
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/textencoders/TextEncoders.jl b/src/textencoders/TextEncoders.jl
@@ -55,6 +55,9 @@ TransformerTextEncoder(v::WList, args...; kws...) = TransformerTextEncoder(TextT
 TransformerTextEncoder(t::AbstractTokenization, v::WList, args...; kws...) =
     TransformerTextEncoder(TextTokenizer(t), v, args...; kws...)
 
+TransformerTextEncoder(tkr::AbstractTokenizer, v::WList, args...; kws...) =
+    throw(MethodError(TransformerTextEncoder, (tkr, v, args...)))
+
 function TransformerTextEncoder(tkr::AbstractTokenizer, words::AbstractVector, process; trunc = nothing,
                                 startsym = "<s>", endsym = "</s>", unksym = "<unk>", padsym = "<pad>")
     vocab_list = copy(words)