diff --git a/operators/tokenizer/bpe_utils.hpp b/operators/tokenizer/bpe_utils.hpp index e1ac85e6..4e1e4d91 100644 --- a/operators/tokenizer/bpe_utils.hpp +++ b/operators/tokenizer/bpe_utils.hpp @@ -198,7 +198,8 @@ class TokenWithRegularExp { std::wsmatch match; if (std::regex_search(wstr, match, pattern)) { - std::u32string_view token = std::u32string_view(W2Ustring(match.str()).data(), match.str().size()); + std::u32string temp = W2Ustring(match.str()); + std::u32string_view token = std::u32string_view(temp.data(), match.str().size()); m_text = std::u32string(match.suffix().first, match.suffix().second); // Update text to the remaining part after the match return token; } else {