From c6e43d14616d6be4b8cccb434202340c781fca07 Mon Sep 17 00:00:00 2001 From: Aliaksei Andreyeu Date: Wed, 1 Jul 2020 17:56:26 -0700 Subject: [PATCH] Processing memory mappings attachement for mmaped buffer Reviewed By: BurntBrunch Differential Revision: D22144682 fbshipit-source-id: d6d8130d9baa6999ff0d7c42cab83bb4067295b2 --- cpp/mmapbuf/writer/MmapBufferTraceWriter.cpp | 48 +++++++- .../writer/MmapBufferTraceWriterTest.cpp | 110 +++++++++++------- .../profilo/mmapbuf/MmapBufferFileHelper.java | 40 ++----- .../profilo/mmapbuf/MmapBufferManager.java | 15 +++ 4 files changed, 141 insertions(+), 72 deletions(-) diff --git a/cpp/mmapbuf/writer/MmapBufferTraceWriter.cpp b/cpp/mmapbuf/writer/MmapBufferTraceWriter.cpp index b8315a34..f33f8684 100644 --- a/cpp/mmapbuf/writer/MmapBufferTraceWriter.cpp +++ b/cpp/mmapbuf/writer/MmapBufferTraceWriter.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,7 @@ using namespace facebook::profilo::mmapbuf::header; namespace { constexpr int64_t kTriggerEventFlag = 0x0002000000000000L; // 1 << 49 +static constexpr char kMemoryMappingKey[] = "l:s:u:o:s"; void loggerWrite( Logger& logger, @@ -186,6 +188,37 @@ bool copyBufferEntries(TraceBuffer& source, TraceBuffer& dest) { } return processed_count > 0; } + +void processMemoryMappingsFile( + Logger& logger, + std::string& file_path, + int64_t timestamp) { + std::ifstream mappingsFile(file_path); + if (!mappingsFile.is_open()) { + return; + } + + int32_t tid = threadID(); + std::string mappingLine; + while (std::getline(mappingsFile, mappingLine)) { + auto mappingId = logger.write(entries::StandardEntry{ + .type = EntryType::MAPPING, + .tid = tid, + .timestamp = timestamp, + }); + auto keyId = logger.writeBytes( + EntryType::STRING_KEY, + mappingId, + reinterpret_cast(kMemoryMappingKey), + sizeof(kMemoryMappingKey)); + logger.writeBytes( + EntryType::STRING_VALUE, + keyId, + reinterpret_cast(mappingLine.c_str()), + mappingLine.size()); + } +} + } // namespace MmapBufferTraceWriter::MmapBufferTraceWriter( @@ -234,11 +267,12 @@ int64_t MmapBufferTraceWriter::writeTrace( auto entriesCount = mapBufferPrefix->header.size; // Number of additional records we need to log in addition to entries from the - // buffer file + some buffer for long string entries. - constexpr auto kServiceRecordCount = 11 + 10 /* extra buffer */; + // buffer file + memory mappings file records + some buffer for long string + // entries. + constexpr auto kExtraRecordCount = 4096; TraceBufferHolder bufferHolder = - TraceBuffer::allocate(entriesCount + kServiceRecordCount); + TraceBuffer::allocate(entriesCount + kExtraRecordCount); TraceBuffer::Cursor startCursor = bufferHolder->currentHead(); Logger logger( [&bufferHolder]() -> logger::PacketBuffer& { return *bufferHolder; }); @@ -287,6 +321,14 @@ int64_t MmapBufferTraceWriter::writeTrace( std::string(mapBufferPrefix->header.sessionId)); loggerWriteQplTriggerAnnotation( logger, qpl_marker_id, "type", type, timestamp); + + const char* mapsFilename = mapBufferPrefix->header.memoryMapsFilename; + if (mapsFilename[0] != '\0') { + auto lastSlashIdx = dump_path.rfind("/"); + std::string mapsPath = dump_path.substr(0, lastSlashIdx + 1) + mapsFilename; + processMemoryMappingsFile(logger, mapsPath, timestamp); + } + loggerWrite(logger, EntryType::TRACE_END, 0, 0, trace_id, timestamp); TraceWriter writer( diff --git a/cpp/test/mmapbuf/writer/MmapBufferTraceWriterTest.cpp b/cpp/test/mmapbuf/writer/MmapBufferTraceWriterTest.cpp index f3b3249a..43474fa1 100644 --- a/cpp/test/mmapbuf/writer/MmapBufferTraceWriterTest.cpp +++ b/cpp/test/mmapbuf/writer/MmapBufferTraceWriterTest.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,11 @@ constexpr int64_t kTraceRecollectionTimestamp = 1234567; constexpr int32_t kConfigId = 11; constexpr int32_t kBuildId = 17; +static const std::array kMappings = { + "libhwui.so:722580c000:586015DEC7C4DA055D33796D9D793508:186000:491000", + "libart-compiler.so:71987dd000:25CFFF6256F96F117E72005B5318E262:c2000:244000", + "libc.so:7224896000:0965E88D999C749783C8947F9B7937E9:40000:a7000"}; + #define EXPECT_STRING_CONTAINS(haystack, needle) \ EXPECT_PRED_FORMAT2(assertContainsInString, haystack, needle) @@ -90,48 +96,17 @@ class MockCallbacks : public TraceCallbacks { MOCK_METHOD2(onTraceAbort, void(int64_t, AbortReason)); }; -struct StandardEntryWrapper { - StandardEntryWrapper(StandardEntry& standard_entry) - : entry(standard_entry), output(precomputeTraceOutput(entry)) {} - - static std::string precomputeTraceOutput( - const StandardEntry& standard_entry) { - outstream().str(""); - visitor().visit(standard_entry); - return outstream().str(); - } - - static std::stringstream& outstream() { - static std::stringstream ss{}; - return ss; - } - - static EntryVisitor& visitor() { - // Replicating part of visitor chain to get the expected output - static PrintEntryVisitor printVisitor(outstream()); - static DeltaEncodingVisitor deltaVisitor(printVisitor); - static TimestampTruncatingVisitor timestampVisitor(deltaVisitor, 6); - return timestampVisitor; - } - - StandardEntry entry; - std::string output; -}; - class MmapBufferTraceWriterTest : public ::testing::Test { protected: MmapBufferTraceWriterTest() : ::testing::Test(), loggedEntries_(), temp_dump_file_("test_dump"), - temp_trace_folder_(kTraceFolder) { + temp_trace_folder_(kTraceFolder), + temp_mappings_file_("maps", temp_dump_file_.path().parent_path()) { std::srand(std::time(nullptr)); } - int dumpFd() { - return temp_dump_file_.fd(); - } - const std::string& dumpPath() { return temp_dump_file_.path().generic_string(); } @@ -166,13 +141,22 @@ class MmapBufferTraceWriterTest : public ::testing::Test { } void writeRandomEntries(TraceBuffer& buf, int records_count) { + std::stringstream outstream{}; + PrintEntryVisitor printVisitor(outstream); + DeltaEncodingVisitor deltaVisitor(printVisitor); + TimestampTruncatingVisitor visitor(deltaVisitor, 6); + Logger logger([&buf]() -> PacketBuffer& { return buf; }); // Write the main service entry before the main content auto serviceEntry = generateTraceBackwardsEntry(); - loggedEntries_.push_back(StandardEntryWrapper(serviceEntry)); + outstream.str(""); + visitor.visit(serviceEntry); + loggedEntries_.push_back(outstream.str()); while (records_count > 0) { auto entry = generateRandomEntry(); - loggedEntries_.push_back(StandardEntryWrapper(entry)); + outstream.str(""); + visitor.visit(entry); + loggedEntries_.push_back(outstream.str()); logger.write(std::move(entry)); --records_count; } @@ -182,13 +166,20 @@ class MmapBufferTraceWriterTest : public ::testing::Test { writeTraceWithRandomEntries(records_count, records_count); } - void writeTraceWithRandomEntries(int records_count, int buffer_size) { + void writeTraceWithRandomEntries( + int records_count, + int buffer_size, + bool set_mappings_file = false) { MmapBufferManager bufManager{}; MmapBufferManagerTestAccessor bufManagerAccessor(bufManager); bool res = bufManager.allocateBuffer(buffer_size, dumpPath(), 1, 1); + ASSERT_EQ(res, true) << "Unable to allocate the buffer"; bufManager.updateHeader(0, kQplId, kTraceId); bufManager.updateId("272c3f80-f076-5a89-e265-60dcf407373b"); - ASSERT_EQ(res, true) << "Unable to allocate the buffer"; + if (set_mappings_file) { + bufManager.updateMemoryMappingFilename( + temp_mappings_file_.path().filename().generic_string()); + } TraceBufferHolder ringBuffer = TraceBuffer::allocateAt( buffer_size, bufManagerAccessor.mmapBufferPointer()); writeRandomEntries(*ringBuffer, records_count); @@ -198,6 +189,15 @@ class MmapBufferTraceWriterTest : public ::testing::Test { << "Unable to msync the buffer: " << strerror(errno); } + void writeMemoryMappingsFile() { + // use output stream to write the content + std::ofstream mappingsFile(temp_mappings_file_.path().generic_string()); + for (const std::string& map : kMappings) { + mappingsFile << map << std::endl; + } + mappingsFile.close(); + } + fs::path getOnlyTraceFile() { auto dir_iter = fs::recursive_directory_iterator(traceFolderPath()); auto is_file = [](const fs::directory_entry& x) { @@ -227,16 +227,24 @@ class MmapBufferTraceWriterTest : public ::testing::Test { void verifyLogEntriesFromTraceFile() { auto traceContents = getOnlyTraceFileContents(); - for (auto entry : loggedEntries_) { + for (std::string& entry : loggedEntries_) { // Compare entry output ignoring first entry id (before "|") - auto entryOutput = entry.output.substr(entry.output.find("|")); + auto entryOutput = entry.substr(entry.find("|")); EXPECT_STRING_CONTAINS(traceContents, entryOutput); } } - std::vector loggedEntries_; + void verifyMemoryMappingEntries() { + auto traceContents = getOnlyTraceFileContents(); + for (const std::string& map : kMappings) { + EXPECT_STRING_CONTAINS(traceContents, map); + } + } + + std::vector loggedEntries_; test::TemporaryFile temp_dump_file_; test::TemporaryDirectory temp_trace_folder_; + test::TemporaryFile temp_mappings_file_; }; TEST_F(MmapBufferTraceWriterTest, testDumpWriteAndRecollectEndToEnd) { @@ -257,6 +265,28 @@ TEST_F(MmapBufferTraceWriterTest, testDumpWriteAndRecollectEndToEnd) { verifyLogEntriesFromTraceFile(); } +TEST_F( + MmapBufferTraceWriterTest, + testDumpWriteAndRecollectEndToEndWithMappings) { + using ::testing::_; + writeTraceWithRandomEntries(10, 10, true); + writeMemoryMappingsFile(); + + std::string testFolder(traceFolderPath()); + std::string testTracePrefix(kTracePrefix); + auto mockCallbacks = std::make_shared<::testing::NiceMock>(); + MmapBufferTraceWriter traceWriter( + testFolder, testTracePrefix, 0, mockCallbacks); + + EXPECT_CALL(*mockCallbacks, onTraceStart(kTraceId, 0, _)); + EXPECT_CALL(*mockCallbacks, onTraceEnd(kTraceId)); + + traceWriter.writeTrace(dumpPath(), "test", kTraceRecollectionTimestamp); + + verifyLogEntriesFromTraceFile(); + verifyMemoryMappingEntries(); +} + TEST_F( MmapBufferTraceWriterTest, testAbortCallbackIsCalledWhenWriterThrowsException) { diff --git a/java/main/com/facebook/profilo/mmapbuf/MmapBufferFileHelper.java b/java/main/com/facebook/profilo/mmapbuf/MmapBufferFileHelper.java index 7eaddd5f..773af0d6 100644 --- a/java/main/com/facebook/profilo/mmapbuf/MmapBufferFileHelper.java +++ b/java/main/com/facebook/profilo/mmapbuf/MmapBufferFileHelper.java @@ -16,34 +16,18 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.util.Arrays; -import java.util.Comparator; +import java.util.List; import javax.annotation.Nullable; class MmapBufferFileHelper { + interface FileDeletionBlacklist { + List getFilenames(); + } + public static final String BUFFER_FILE_SUFFIX = ".buff"; public static final String MEMORY_MAPPING_FILE_SUFFIX = ".maps"; - private static final int MAX_DUMPS_TO_KEEP = 3; - - private static final Comparator MOST_RECENT_FILES_COMPARATOR = - new Comparator() { - @Override - public int compare(File f1, File f2) { - long f1LastModified = f1.lastModified(); - long f2LastModified = f2.lastModified(); - if (f1LastModified == f2LastModified) { - return 0; - } - if (f1LastModified < f2LastModified) { - return 1; - } else { - return -1; - } - } - }; - private final File mMmapFilesFolder; public static final Object DUMP_FILES_LOCK = new Object(); @@ -72,7 +56,7 @@ public boolean accept(File dir, String name) { return foundFiles[0]; } - public void deleteOldBufferFiles() { + public void deleteOldBufferFiles(@Nullable FileDeletionBlacklist blacklist) { File mmapFolder = getFolderIfExists(); if (mmapFolder == null) { return; @@ -81,13 +65,11 @@ public void deleteOldBufferFiles() { if (mmapFiles == null) { return; } - int filesCount = mmapFiles.length; - if (filesCount <= MAX_DUMPS_TO_KEEP) { - return; - } - Arrays.sort(mmapFiles, MOST_RECENT_FILES_COMPARATOR); - for (int i = MAX_DUMPS_TO_KEEP; i < filesCount; i++) { - File file = mmapFiles[i]; + List blacklistFilenames = blacklist.getFilenames(); + for (File file : mmapFiles) { + if (blacklistFilenames.contains(file.getName())) { + continue; + } synchronized (DUMP_FILES_LOCK) { if (file.exists()) { file.delete(); diff --git a/java/main/com/facebook/profilo/mmapbuf/MmapBufferManager.java b/java/main/com/facebook/profilo/mmapbuf/MmapBufferManager.java index ded74cdd..8f91ac1d 100644 --- a/java/main/com/facebook/profilo/mmapbuf/MmapBufferManager.java +++ b/java/main/com/facebook/profilo/mmapbuf/MmapBufferManager.java @@ -21,6 +21,9 @@ import com.facebook.jni.annotations.DoNotStrip; import com.facebook.soloader.SoLoader; import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; @@ -60,6 +63,18 @@ public MmapBufferManager(long configId, File folder, Context context) { return mMmapFileName; } + public List getCurrentFilenames() { + if (mMmapFileName == null) { + return Collections.emptyList(); + } + ArrayList filenames = new ArrayList<>(2); + filenames.add(mMmapFileName); + if (mMemoryMappingsFile != null) { + filenames.add(mMemoryMappingsFile.getName()); + } + return filenames; + } + public boolean isEnabled() { return mEnabled.get(); }