diff --git a/core/processor/ProcessorSplitMultilineLogStringNative.cpp b/core/processor/ProcessorSplitMultilineLogStringNative.cpp index dbd45f30f7..12df049b34 100644 --- a/core/processor/ProcessorSplitMultilineLogStringNative.cpp +++ b/core/processor/ProcessorSplitMultilineLogStringNative.cpp @@ -348,7 +348,7 @@ void ProcessorSplitMultilineLogStringNative::HandleUnmatchLogs(const StringView& EventsContainer& newEvents, StringView logPath, int* unmatchLines) { - size_t begin, fisrtLogSize, totalLines = 0; + size_t begin = 0, fisrtLogSize = 0, totalLines = 0; while (begin < sourceVal.size()) { StringView content = GetNextLine(sourceVal, begin); ++(*unmatchLines); diff --git a/core/reader/JsonLogFileReader.cpp b/core/reader/JsonLogFileReader.cpp index 3d1aeb6cbb..0796b1abca 100644 --- a/core/reader/JsonLogFileReader.cpp +++ b/core/reader/JsonLogFileReader.cpp @@ -19,8 +19,10 @@ using namespace std; namespace logtail { -int32_t -JsonLogFileReader::LastMatchedLine(char* buffer, int32_t size, int32_t& rollbackLineFeedCount, bool allowRollback) { +int32_t JsonLogFileReader::RemoveLastIncompleteLog(char* buffer, + int32_t size, + int32_t& rollbackLineFeedCount, + bool allowRollback) { int32_t readBytes = 0; int32_t endIdx = 0; int32_t beginIdx = 0; diff --git a/core/reader/JsonLogFileReader.h b/core/reader/JsonLogFileReader.h index 6df04229cc..c8bbc8a3e9 100644 --- a/core/reader/JsonLogFileReader.h +++ b/core/reader/JsonLogFileReader.h @@ -30,8 +30,10 @@ class JsonLogFileReader : public LogFileReader { : LogFileReader(hostLogPathDir, hostLogPathFile, devInode, readerConfig, multilineConfig) {} protected: - int32_t - LastMatchedLine(char* buffer, int32_t size, int32_t& rollbackLineFeedCount, bool allowRollback = true) override; + int32_t RemoveLastIncompleteLog(char* buffer, + int32_t size, + int32_t& rollbackLineFeedCount, + bool allowRollback = true) override; private: bool FindJsonMatch( @@ -40,7 +42,7 @@ class JsonLogFileReader : public LogFileReader { #ifdef APSARA_UNIT_TEST_MAIN friend class JsonLogFileReaderUnittest; friend class JsonParseLogLineUnittest; - friend class LastMatchedLineUnittest; + friend class RemoveLastIncompleteLogUnittest; #endif }; diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index c4144b7045..a10d2e3371 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -1698,7 +1698,7 @@ void LogFileReader::ReadUTF8(LogBuffer& logBuffer, int64_t end, bool& moreData, } if (allowRollback || mReaderConfig.second->RequiringJsonReader()) { int32_t rollbackLineFeedCount; - nbytes = LastMatchedLine(stringBuffer, alignedBytes, rollbackLineFeedCount, allowRollback); + nbytes = RemoveLastIncompleteLog(stringBuffer, alignedBytes, rollbackLineFeedCount, allowRollback); } if (nbytes == 0) { @@ -1706,7 +1706,7 @@ void LogFileReader::ReadUTF8(LogBuffer& logBuffer, int64_t end, bool& moreData, nbytes = alignedBytes ? alignedBytes : BUFFER_SIZE; if (mReaderConfig.second->RequiringJsonReader()) { int32_t rollbackLineFeedCount; - nbytes = LastMatchedLine(stringBuffer, nbytes, rollbackLineFeedCount, false); + nbytes = RemoveLastIncompleteLog(stringBuffer, nbytes, rollbackLineFeedCount, false); } LOG_WARNING(sLogger, ("Log is too long and forced to be split at offset: ", @@ -1856,7 +1856,7 @@ void LogFileReader::ReadGBK(LogBuffer& logBuffer, int64_t end, bool& moreData, b int32_t rollbackLineFeedCount = 0; int32_t bakResultCharCount = resultCharCount; if (allowRollback || mReaderConfig.second->RequiringJsonReader()) { - resultCharCount = LastMatchedLine(stringBuffer, resultCharCount, rollbackLineFeedCount, allowRollback); + resultCharCount = RemoveLastIncompleteLog(stringBuffer, resultCharCount, rollbackLineFeedCount, allowRollback); } if (resultCharCount == 0) { if (moreData) { @@ -1864,7 +1864,7 @@ void LogFileReader::ReadGBK(LogBuffer& logBuffer, int64_t end, bool& moreData, b rollbackLineFeedCount = 0; if (mReaderConfig.second->RequiringJsonReader()) { int32_t rollbackLineFeedCount; - LastMatchedLine(stringBuffer, resultCharCount, rollbackLineFeedCount, false); + RemoveLastIncompleteLog(stringBuffer, resultCharCount, rollbackLineFeedCount, false); } // Cannot get the split position here, so just mark a flag and send alarm later logTooLongSplitFlag = true; @@ -2027,74 +2027,75 @@ LogFileReader::FileCompareResult LogFileReader::CompareToFile(const string& file 1. xxx\nend\n -> xxx\nend 1. xxx\nend\nxxx\n -> xxx\nend */ -int32_t LogFileReader::LastMatchedLine(char* buffer, int32_t size, int32_t& rollbackLineFeedCount, bool allowRollback) { +/* + return: the number of bytes left, including \n +*/ +int32_t +LogFileReader::RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& rollbackLineFeedCount, bool allowRollback) { if (!allowRollback) { return size; } - int endPs = size - 1; // buffer[size] = 0 , buffer[size-1] = '\n' - rollbackLineFeedCount = 0; - // Single line rollback - if (!mMultilineConfig.first->IsMultiline()) { - while (endPs >= 0) { - if (buffer[endPs] == '\n') { - if (endPs != size - 1) { // if last line dose not end with '\n', rollback - ++rollbackLineFeedCount; - } - return endPs + 1; - } - endPs--; - } - return 0; + int32_t endPs; // the position of \n or \0 + if (buffer[size - 1] == '\n') { + endPs = size - 1; + } else { + endPs = size; } + rollbackLineFeedCount = 0; // Multiline rollback - int begPs = size - 2; - std::string exception; - while (begPs >= 0) { - if (buffer[begPs] == '\n' || begPs == 0) { - int lineBegin = begPs == 0 ? 0 : begPs + 1; - if (mMultilineConfig.first->GetContinuePatternReg() - && BoostRegexMatch(buffer + lineBegin, - endPs - lineBegin, - *mMultilineConfig.first->GetContinuePatternReg(), - exception)) { - ++rollbackLineFeedCount; - endPs = begPs; - } else if (mMultilineConfig.first->GetEndPatternReg() - && BoostRegexMatch(buffer + lineBegin, - endPs - lineBegin, - *mMultilineConfig.first->GetEndPatternReg(), - exception)) { - // Ensure the end line is complete - if (buffer[endPs] == '\n') { - return endPs + 1; - } else { - ++rollbackLineFeedCount; - endPs = begPs; + if (mMultilineConfig.first->IsMultiline()) { + std::string exception; + while (endPs >= 0) { + StringView content = GetLastLine(StringView(buffer, size), endPs); + if (mMultilineConfig.first->GetEndPatternReg()) { + // start + end, continue + end, end + if (BoostRegexMatch( + content.data(), content.size(), *mMultilineConfig.first->GetEndPatternReg(), exception)) { + // Ensure the end line is complete + if (buffer[endPs] == '\n') { + return endPs + 1; + } } } else if (mMultilineConfig.first->GetStartPatternReg() - && BoostRegexMatch(buffer + lineBegin, - endPs - lineBegin, - *mMultilineConfig.first->GetStartPatternReg(), - exception)) { + && BoostRegexMatch( + content.data(), content.size(), *mMultilineConfig.first->GetStartPatternReg(), exception)) { + // start + continue, start ++rollbackLineFeedCount; // Keep all the buffer if rollback all - return lineBegin; - } else if (mMultilineConfig.first->GetContinuePatternReg()) { - // We can confirm the logs before are complete if continue is configured but no regex pattern can match. - if (buffer[endPs] == '\n') { - return endPs + 1; - } else { - // Keep all the buffer if rollback all - return lineBegin; - } - } else { - ++rollbackLineFeedCount; - endPs = begPs; + return content.data() - buffer; } + ++rollbackLineFeedCount; + endPs = content.data() - buffer - 1; } - begPs--; } - return 0; + // Single line rollback or all unmatch rollback + rollbackLineFeedCount = 0; + if (buffer[size - 1] == '\n') { + return size; + } + StringView content = GetLastLine(StringView(buffer, size), size - 1); + ++rollbackLineFeedCount; + return content.data() - buffer; +} + +/* + params: + buffer: all read logs + end: the end position of current line, \n or \0 + return: + last line (backward), without \n or \0 +*/ +StringView LogFileReader::GetLastLine(StringView buffer, size_t end) { + if (end == 0) { + return buffer; + } + + for (size_t begin = end; begin > 0; --begin) { + if (buffer[begin - 1] == '\n') { + return StringView(buffer.data() + begin, end - begin); + } + } + return StringView(buffer.data(), end); } size_t LogFileReader::AlignLastCharacter(char* buffer, size_t size) { @@ -2108,8 +2109,8 @@ size_t LogFileReader::AlignLastCharacter(char* buffer, size_t size) { // 1. The number of byte for one character can be 1, 2, 4. // 2. 1 byte character: the top bit is 0. // 3. 2 bytes character: the 1st byte is between 0x81 and 0xFE; the 2nd byte is between 0x40 and 0xFE. - // 4. 4 bytes character: the 1st and 3rd byte is between 0x81 and 0xFE; the 2nd and 4th byte are between 0x30 - // and 0x39. (not supported to align) + // 4. 4 bytes character: the 1st and 3rd byte is between 0x81 and 0xFE; the 2nd and 4th byte are between + // 0x30 and 0x39. (not supported to align) // 1 byte character, 2nd byte of 2 bytes, 2nd or 4th byte of 4 bytes if ((buffer[endPs] & 0x80) == 0 || size == 1) { diff --git a/core/reader/LogFileReader.h b/core/reader/LogFileReader.h index 9ed95de06c..11586122cb 100644 --- a/core/reader/LogFileReader.h +++ b/core/reader/LogFileReader.h @@ -128,7 +128,7 @@ class LogFileReader { FileCompareResult CompareToFile(const std::string& filePath); virtual int32_t - LastMatchedLine(char* buffer, int32_t size, int32_t& rollbackLineFeedCount, bool allowRollback = true); + RemoveLastIncompleteLog(char* buffer, int32_t size, int32_t& rollbackLineFeedCount, bool allowRollback = true); size_t AlignLastCharacter(char* buffer, size_t size); @@ -502,6 +502,8 @@ class LogFileReader { // @param fromCpt: if the read size is recoveried from checkpoint, set it to true. size_t getNextReadSize(int64_t fileEnd, bool& fromCpt); + StringView GetLastLine(StringView buffer, size_t end); + // Update current checkpoint's read offset and length after success read. void setExactlyOnceCheckpointAfterRead(size_t readSize); @@ -587,8 +589,7 @@ class LogFileReader { friend class LogSplitUnittest; friend class LogSplitDiscardUnmatchUnittest; friend class LogSplitNoDiscardUnmatchUnittest; - friend class LastMatchedLineDiscardUnmatchUnittest; - friend class LastMatchedLineNoDiscardUnmatchUnittest; + friend class RemoveLastIncompleteLogMultilineUnittest; friend class LogFileReaderCheckpointUnittest; protected: diff --git a/core/unittest/reader/CMakeLists.txt b/core/unittest/reader/CMakeLists.txt index c57cfa4c5c..22370e9c3a 100644 --- a/core/unittest/reader/CMakeLists.txt +++ b/core/unittest/reader/CMakeLists.txt @@ -24,7 +24,7 @@ target_link_libraries(file_reader_options_unittest unittest_base) add_executable(json_log_file_reader_unittest JsonLogFileReaderUnittest.cpp) target_link_libraries(json_log_file_reader_unittest unittest_base) -add_executable(last_matched_line_unittest LastMatchedLineUnittest.cpp) +add_executable(last_matched_line_unittest RemoveLastIncompleteLogUnittest.cpp) target_link_libraries(last_matched_line_unittest unittest_base) add_executable(log_file_reader_unittest LogFileReaderUnittest.cpp) diff --git a/core/unittest/reader/JsonLogFileReaderUnittest.cpp b/core/unittest/reader/JsonLogFileReaderUnittest.cpp index b673f593a9..2e1e7e0d65 100644 --- a/core/unittest/reader/JsonLogFileReaderUnittest.cpp +++ b/core/unittest/reader/JsonLogFileReaderUnittest.cpp @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" #include + #include -#include "reader/JsonLogFileReader.h" -#include "common/RuntimeUtil.h" + #include "common/FileSystemUtil.h" +#include "common/RuntimeUtil.h" #include "file_server/FileServer.h" +#include "reader/JsonLogFileReader.h" +#include "unittest/Unittest.h" DECLARE_FLAG_INT32(force_release_deleted_file_fd_timeout); @@ -227,21 +229,21 @@ void JsonLogFileReaderUnittest::TestReadUTF8() { } } -class LastMatchedLineUnittest : public ::testing::Test { +class RemoveLastIncompleteLogUnittest : public ::testing::Test { public: void SetUp() override { mLogFileReader.reset(new JsonLogFileReader( "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx))); } - void TestLastMatchedLineSingleLine(); - void TestLastMatchedLineSingleLineIncomplete(); - void TestLastMatchedLineSingleLineIncompleteNoRollback(); - void TestLastMatchedLineMultiline(); - void TestLastMatchedLineMultilineIncomplete(); - void TestLastMatchedLineMultilineIncompleteNoRollback(); - void TestLastMatchedLineNotValidJson(); - void TestLastMatchedLineNotValidJsonNoRollback(); + void TestRemoveLastIncompleteLogSingleLine(); + void TestRemoveLastIncompleteLogSingleLineIncomplete(); + void TestRemoveLastIncompleteLogSingleLineIncompleteNoRollback(); + void TestRemoveLastIncompleteLogMultiline(); + void TestRemoveLastIncompleteLogMultilineIncomplete(); + void TestRemoveLastIncompleteLogMultilineIncompleteNoRollback(); + void TestRemoveLastIncompleteLogNotValidJson(); + void TestRemoveLastIncompleteLogNotValidJsonNoRollback(); std::unique_ptr mLogFileReader; MultilineOptions multilineOpts; @@ -249,16 +251,16 @@ class LastMatchedLineUnittest : public ::testing::Test { PipelineContext ctx; }; -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineSingleLine) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineSingleLineIncomplete) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineSingleLineIncompleteNoRollback) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineMultiline) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineMultilineIncomplete) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineMultilineIncompleteNoRollback) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineNotValidJson) -UNIT_TEST_CASE(LastMatchedLineUnittest, TestLastMatchedLineNotValidJsonNoRollback) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogSingleLine) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogSingleLineIncomplete) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogSingleLineIncompleteNoRollback) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogMultiline) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogMultilineIncomplete) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogMultilineIncompleteNoRollback) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogNotValidJson) +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestRemoveLastIncompleteLogNotValidJsonNoRollback) -void LastMatchedLineUnittest::TestLastMatchedLineSingleLine() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogSingleLine() { { // case single line std::string line1 = R"({"key": "first value"})"; std::string line2 = R"({"key": "second value"})"; @@ -266,15 +268,15 @@ void LastMatchedLineUnittest::TestLastMatchedLineSingleLine() { std::string expectMatch = line1 + '\0' + line2 + '\0' + line3 + '\0'; std::string testLog = line1 + '\n' + line2 + '\n' + line3 + '\n'; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } } -void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncomplete() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogSingleLineIncomplete() { { // case single line, buffer size not big enough, json truncated std::string line1 = R"({"key": "first value"})"; std::string line2 = R"({"key": "second value"})"; @@ -282,8 +284,8 @@ void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncomplete() { std::string expectMatch = line1 + '\0' + line2 + '\0'; std::string testLog = line1 + '\n' + line2 + '\n' + line3; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); @@ -295,15 +297,15 @@ void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncomplete() { std::string expectMatch = line1 + '\0' + line2 + '\0'; std::string testLog = line1 + '\n' + line2 + '\n' + line3; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } } -void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncompleteNoRollback() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogSingleLineIncompleteNoRollback() { { // case single line, buffer size not big enough, json truncated std::string line1 = R"({"key": "first value"})"; std::string line2 = R"({"key": "second value"})"; @@ -311,7 +313,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncompleteNoRollback( std::string expectMatch = line1 + '\0' + line2 + '\0' + line3; std::string testLog = line1 + '\n' + line2 + '\n' + line3; int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); @@ -324,7 +326,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncompleteNoRollback( std::string expectMatch = line1 + '\0' + line2 + '\0' + line3; std::string testLog = line1 + '\n' + line2 + '\n' + line3; int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); @@ -332,7 +334,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineSingleLineIncompleteNoRollback( } } -void LastMatchedLineUnittest::TestLastMatchedLineMultiline() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogMultiline() { { // case multi line std::vector index; std::string firstLog = R"({ @@ -346,15 +348,15 @@ void LastMatchedLineUnittest::TestLastMatchedLineMultiline() { std::string expectMatch = firstLog + '\0' + secondLog + '\0'; std::string testLog = firstLog + '\n' + secondLog + '\n'; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } } -void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncomplete() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogMultilineIncomplete() { { // case multi line, buffer size not enough, json truncated std::vector index; std::string firstLog = R"({ @@ -367,8 +369,8 @@ void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncomplete() { std::string expectMatch = firstLog + '\0'; std::string testLog = firstLog + '\n' + secondLog + '\n'; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(4, rollbackLineFeedCount); @@ -386,15 +388,15 @@ void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncomplete() { std::string expectMatch = firstLog + '\0'; std::string testLog = firstLog + '\n' + secondLog; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(5, rollbackLineFeedCount); } } -void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncompleteNoRollback() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogMultilineIncompleteNoRollback() { { // case multi line, buffer size not enough, json truncated std::vector index; std::string firstLog = R"({ @@ -409,7 +411,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncompleteNoRollback() std::string expectMatch = firstLog + '\0' + splittedSecondLog + '\0'; std::string testLog = firstLog + '\n' + secondLog + '\n'; int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); @@ -428,7 +430,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncompleteNoRollback() std::string expectMatch = firstLog + '\0' + secondLog; std::string testLog = firstLog + '\n' + secondLog; int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); @@ -436,12 +438,12 @@ void LastMatchedLineUnittest::TestLastMatchedLineMultilineIncompleteNoRollback() } } -void LastMatchedLineUnittest::TestLastMatchedLineNotValidJson() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogNotValidJson() { { // case not json, skip all std::string testLog = "not a json at all.\nnot a json at all.\nnot a json at all.\n"; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(testLog.size(), matchSize); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } @@ -451,8 +453,8 @@ void LastMatchedLineUnittest::TestLastMatchedLineNotValidJson() { std::replace(expectMatch.begin(), expectMatch.end(), '\n', '\0'); std::string testLog = "not a json at all.\nnot a json at all.\n{partial json\n"; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); @@ -469,19 +471,19 @@ void LastMatchedLineUnittest::TestLastMatchedLineNotValidJson() { std::string expectMatch = firstLog + '\0' + notjson; ; int32_t rollbackLineFeedCount = 0; - size_t matchSize - = mLogFileReader->LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } } -void LastMatchedLineUnittest::TestLastMatchedLineNotValidJsonNoRollback() { +void RemoveLastIncompleteLogUnittest::TestRemoveLastIncompleteLogNotValidJsonNoRollback() { { // case not json std::string testLog = "not a json at all.\nnot a json at all.\nnot a json at all.\n"; int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(testLog.size(), matchSize); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); @@ -491,7 +493,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineNotValidJsonNoRollback() { std::string expectMatch = testLog; std::replace(expectMatch.begin(), expectMatch.end(), '\n', '\0'); int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); @@ -509,7 +511,7 @@ void LastMatchedLineUnittest::TestLastMatchedLineNotValidJsonNoRollback() { std::string expectMatch = firstLog + '\0' + notjson; ; int32_t rollbackLineFeedCount = 0; - size_t matchSize = mLogFileReader->LastMatchedLine( + size_t matchSize = mLogFileReader->RemoveLastIncompleteLog( const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount, false); APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); diff --git a/core/unittest/reader/LastMatchedLineUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp similarity index 51% rename from core/unittest/reader/LastMatchedLineUnittest.cpp rename to core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 11cbc1d6ee..1907372e46 100644 --- a/core/unittest/reader/LastMatchedLineUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "unittest/Unittest.h" +#include "common/FileSystemUtil.h" #include "reader/LogFileReader.h" #include "reader/SourceBuffer.h" -#include "common/FileSystemUtil.h" +#include "unittest/Unittest.h" namespace logtail { @@ -27,7 +27,7 @@ const std::string LOG_END_STRING = " ...23 more"; const std::string LOG_END_REGEX = R"(\s*\.\.\.\d+ more)"; const std::string LOG_UNMATCH = "unmatch log"; -class LastMatchedLineUnittest : public ::testing::Test { +class RemoveLastIncompleteLogUnittest : public ::testing::Test { public: static void SetUpTestCase() { logPathDir = GetProcessExecutionDir(); @@ -75,14 +75,14 @@ class LastMatchedLineUnittest : public ::testing::Test { static std::string utf8File; }; -UNIT_TEST_CASE(LastMatchedLineUnittest, TestSingleline); -UNIT_TEST_CASE(LastMatchedLineUnittest, TestMultiline); +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestSingleline); +UNIT_TEST_CASE(RemoveLastIncompleteLogUnittest, TestMultiline); -std::string LastMatchedLineUnittest::logPathDir; -std::string LastMatchedLineUnittest::gbkFile; -std::string LastMatchedLineUnittest::utf8File; +std::string RemoveLastIncompleteLogUnittest::logPathDir; +std::string RemoveLastIncompleteLogUnittest::gbkFile; +std::string RemoveLastIncompleteLogUnittest::utf8File; -void LastMatchedLineUnittest::TestSingleline() { +void RemoveLastIncompleteLogUnittest::TestSingleline() { MultilineOptions multilineOpts; LogFileReader logFileReader( logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); @@ -93,8 +93,8 @@ void LastMatchedLineUnittest::TestSingleline() { std::string expectMatch = line1 + '\n' + line2 + '\n' + line3 + '\n'; std::string testLog = expectMatch; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); @@ -106,24 +106,25 @@ void LastMatchedLineUnittest::TestSingleline() { std::string expectMatch = line1 + '\n' + line2 + '\n'; std::string testLog = expectMatch + line3; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { // case single line, cannot be split, buffer size not big enough (no new line at the end of line) + // it will be force read in ReadUTF8/ReadGBK std::string testLog = "first."; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(0, matchSize); // return the whole buffer, so no rollback - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } } -void LastMatchedLineUnittest::TestMultiline() { +void RemoveLastIncompleteLogUnittest::TestMultiline() { Json::Value config; config["StartPattern"] = LOG_BEGIN_REGEX; MultilineOptions multilineOpts; @@ -137,8 +138,8 @@ void LastMatchedLineUnittest::TestMultiline() { std::string secondLog = LOG_BEGIN_STRING + "second.\nmultiline1\nmultiline2"; std::string expectMatch = firstLog + '\n'; std::string testLog = expectMatch + secondLog + '\n'; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); @@ -149,48 +150,49 @@ void LastMatchedLineUnittest::TestMultiline() { std::string secondLog = LOG_BEGIN_STRING + "second.\nmultiline1\nmultiline2"; std::string expectMatch = firstLog + '\n'; std::string testLog = expectMatch + secondLog; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); } { // case multi line not match std::string testLog2 = "log begin does not match.\nlog begin does not match.\nlog begin does not match.\n"; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog2.data()), testLog2.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(0, matchSize); - APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog2.data()), testLog2.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(testLog2.size(), matchSize); + APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } { // case multi line not match, buffer size not big enough (no new line at the end of line) - std::string testLog2 = "log begin does not match.\nlog begin does not match.\nlog begin does not"; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog2.data()), testLog2.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(0, matchSize); - APSARA_TEST_EQUAL_FATAL(3, rollbackLineFeedCount); + std::string expectMatch = "log begin does not match.\nlog begin does not match.\n"; + std::string testLog2 = expectMatch + "log begin does not"; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog2.data()), testLog2.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(expectMatch.size(), matchSize); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } } -class LastMatchedLineDiscardUnmatchUnittest : public ::testing::Test { +class RemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { public: - void TestLastMatchedLineWithBeginContinue(); - void TestLastMatchedLineWithBeginEnd(); - void TestLastMatchedLineWithBegin(); - void TestLastMatchedLineWithContinueEnd(); - void TestLastMatchedLineWithEnd(); + void TestRemoveLastIncompleteLogWithBeginContinue(); + void TestRemoveLastIncompleteLogWithBeginEnd(); + void TestRemoveLastIncompleteLogWithBegin(); + void TestRemoveLastIncompleteLogWithContinueEnd(); + void TestRemoveLastIncompleteLogWithEnd(); private: FileReaderOptions readerOpts; PipelineContext ctx; }; -UNIT_TEST_CASE(LastMatchedLineDiscardUnmatchUnittest, TestLastMatchedLineWithBeginContinue); -UNIT_TEST_CASE(LastMatchedLineDiscardUnmatchUnittest, TestLastMatchedLineWithBeginEnd); -UNIT_TEST_CASE(LastMatchedLineDiscardUnmatchUnittest, TestLastMatchedLineWithBegin); -UNIT_TEST_CASE(LastMatchedLineDiscardUnmatchUnittest, TestLastMatchedLineWithContinueEnd); -UNIT_TEST_CASE(LastMatchedLineDiscardUnmatchUnittest, TestLastMatchedLineWithEnd); +UNIT_TEST_CASE(RemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginContinue); +UNIT_TEST_CASE(RemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBeginEnd); +UNIT_TEST_CASE(RemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithBegin); +UNIT_TEST_CASE(RemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithContinueEnd); +UNIT_TEST_CASE(RemoveLastIncompleteLogMultilineUnittest, TestRemoveLastIncompleteLogWithEnd); -void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBeginContinue() { +void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBeginContinue() { Json::Value config; config["StartPattern"] = LOG_BEGIN_REGEX; config["ContinuePattern"] = LOG_CONTINUE_REGEX; @@ -203,8 +205,8 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBeginContinue std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + '\n'; std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(2, rollbackLineFeedCount); @@ -213,26 +215,35 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBeginContinue std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + '\n'; std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { // case: end with unmatch - std::string expectMatch - = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_UNMATCH + "\n"; - std::string testLog = std::string(expectMatch.data()); + std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n"; + std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(2, rollbackLineFeedCount); + } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } } -void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBeginEnd() { +void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBeginEnd() { Json::Value config; config["StartPattern"] = LOG_BEGIN_REGEX; config["EndPattern"] = LOG_END_REGEX; @@ -245,18 +256,18 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBeginEnd() { std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; std::string testLog = std::string(expectMatch.data()); int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } { // case: end with begin - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n"; + std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_BEGIN_STRING + LOG_BEGIN_STRING + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); @@ -265,15 +276,25 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBeginEnd() { std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + "\n"; std::string testLog = expectMatch + LOG_UNMATCH + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } } -void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBegin() { +void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBegin() { Json::Value config; config["StartPattern"] = LOG_BEGIN_REGEX; MultilineOptions multilineOpts; @@ -285,8 +306,8 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBegin() { std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_UNMATCH + '\n'; std::string testLog = expectMatch + LOG_BEGIN_STRING; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); @@ -295,15 +316,25 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithBegin() { std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n"; std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(2, rollbackLineFeedCount); } + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; + int32_t rollbackLineFeedCount = 0; + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); + APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); + APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); + } } -void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithContinueEnd() { +void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithContinueEnd() { Json::Value config; config["ContinuePattern"] = LOG_CONTINUE_REGEX; config["EndPattern"] = LOG_END_REGEX; @@ -316,8 +347,8 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithContinueEnd() std::string expectMatch = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + '\n'; std::string testLog = std::string(expectMatch.data()); int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); @@ -326,248 +357,68 @@ void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithContinueEnd() std::string expectMatch = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + '\n'; std::string testLog = expectMatch + LOG_CONTINUE_STRING + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } { // case: end with unmatch - std::string expectMatch - = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + "\n" + LOG_UNMATCH + "\n"; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } -} - -void LastMatchedLineDiscardUnmatchUnittest::TestLastMatchedLineWithEnd() { - Json::Value config; - config["EndPattern"] = LOG_END_REGEX; - MultilineOptions multilineOpts; - multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - // logFileReader.mDiscardUnmatch = true; - { // case: end with end - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } - { // case: end with unmatch - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string expectMatch = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + "\n"; std::string testLog = expectMatch + LOG_UNMATCH + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } -} - -class LastMatchedLineNoDiscardUnmatchUnittest : public ::testing::Test { -public: - void TestLastMatchedLineWithBeginContinue(); - void TestLastMatchedLineWithBeginEnd(); - void TestLastMatchedLineWithBegin(); - void TestLastMatchedLineWithContinueEnd(); - void TestLastMatchedLineWithEnd(); - -private: - FileReaderOptions readerOpts; - PipelineContext ctx; -}; - -UNIT_TEST_CASE(LastMatchedLineNoDiscardUnmatchUnittest, TestLastMatchedLineWithBeginContinue); -UNIT_TEST_CASE(LastMatchedLineNoDiscardUnmatchUnittest, TestLastMatchedLineWithBeginEnd); -UNIT_TEST_CASE(LastMatchedLineNoDiscardUnmatchUnittest, TestLastMatchedLineWithBegin); -UNIT_TEST_CASE(LastMatchedLineNoDiscardUnmatchUnittest, TestLastMatchedLineWithContinueEnd); -UNIT_TEST_CASE(LastMatchedLineNoDiscardUnmatchUnittest, TestLastMatchedLineWithEnd); - -void LastMatchedLineNoDiscardUnmatchUnittest::TestLastMatchedLineWithBeginContinue() { - Json::Value config; - config["StartPattern"] = LOG_BEGIN_REGEX; - config["ContinuePattern"] = LOG_CONTINUE_REGEX; - MultilineOptions multilineOpts; - multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - { // case: end with begin continue - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + '\n'; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(2, rollbackLineFeedCount); - } - { // case: end with begin - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + '\n'; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n"; + { // case: only \n + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } - { // case: end with unmatch - std::string expectMatch - = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_UNMATCH + "\n"; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } } -void LastMatchedLineNoDiscardUnmatchUnittest::TestLastMatchedLineWithBeginEnd() { +void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEnd() { Json::Value config; - config["StartPattern"] = LOG_BEGIN_REGEX; config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); LogFileReader logFileReader( "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - { // case: end with begin end - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + // logFileReader.mDiscardUnmatch = true; + { // case: end with end + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; std::string testLog = std::string(expectMatch.data()); int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); } - { // case: end with begin - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + '\n'; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); - } - { // case: end with unmatch - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + "\n"; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(2, rollbackLineFeedCount); - } -} - -void LastMatchedLineNoDiscardUnmatchUnittest::TestLastMatchedLineWithBegin() { - Json::Value config; - config["StartPattern"] = LOG_BEGIN_REGEX; - MultilineOptions multilineOpts; - multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - { // case: end with begin - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_UNMATCH + '\n'; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); - } { // case: end with unmatch - std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n"; - std::string testLog = expectMatch + LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n"; - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(2, rollbackLineFeedCount); - } -} - -void LastMatchedLineNoDiscardUnmatchUnittest::TestLastMatchedLineWithContinueEnd() { - Json::Value config; - config["ContinuePattern"] = LOG_CONTINUE_REGEX; - config["EndPattern"] = LOG_END_REGEX; - MultilineOptions multilineOpts; - multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - { // case: end with continue end - std::string expectMatch = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + '\n'; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } - { // case: end with continue - std::string expectMatch = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + '\n'; - std::string testLog = expectMatch + LOG_CONTINUE_STRING + "\n"; + std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; + std::string testLog = expectMatch + LOG_UNMATCH + "\n"; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount); } - { // case: end with unmatch - std::string expectMatch - = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + "\n" + LOG_UNMATCH + "\n"; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } -} - -void LastMatchedLineNoDiscardUnmatchUnittest::TestLastMatchedLineWithEnd() { - Json::Value config; - config["EndPattern"] = LOG_END_REGEX; - MultilineOptions multilineOpts; - multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); - { // case: end with end - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + "\n"; - std::string testLog = std::string(expectMatch.data()); - int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); - APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); - APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); - APSARA_TEST_EQUAL_FATAL(0, rollbackLineFeedCount); - } - { // case: end with unmatch - std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + "\n"; - std::string testLog = expectMatch + LOG_UNMATCH + "\n"; + { // case: all unmatch + std::string expectMatch = "\n\n"; + std::string testLog = expectMatch + LOG_UNMATCH; int32_t rollbackLineFeedCount = 0; - int32_t matchSize - = logFileReader.LastMatchedLine(const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); + int32_t matchSize = logFileReader.RemoveLastIncompleteLog( + const_cast(testLog.data()), testLog.size(), rollbackLineFeedCount); APSARA_TEST_EQUAL_FATAL(static_cast(expectMatch.size()), matchSize); APSARA_TEST_EQUAL_FATAL(std::string(testLog.data(), matchSize), expectMatch); APSARA_TEST_EQUAL_FATAL(1, rollbackLineFeedCount);