Skip to content

Commit

Permalink
Processing memory mappings attachement for mmaped buffer
Browse files Browse the repository at this point in the history 
Reviewed By: BurntBrunch

Differential Revision: D22144682

fbshipit-source-id: d6d8130d9baa6999ff0d7c42cab83bb4067295b2
  • Loading branch information
@aandreyeu @facebook-github-bot
aandreyeu authored and facebook-github-bot committed Jul 2, 2020
1 parent b6f32ab commit c6e43d1
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 72 deletions.
48 changes: 45 additions & 3 deletions cpp/mmapbuf/writer/MmapBufferTraceWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <cstring>
#include <fstream>
#include <stdexcept>

#include <profilo/entries/EntryType.h>
Expand All @@ -42,6 +43,7 @@ using namespace facebook::profilo::mmapbuf::header;
namespace {

constexpr int64_t kTriggerEventFlag = 0x0002000000000000L; // 1 << 49
static constexpr char kMemoryMappingKey[] = "l:s:u:o:s";

void loggerWrite(
Logger& logger,
Expand Down Expand Up @@ -186,6 +188,37 @@ bool copyBufferEntries(TraceBuffer& source, TraceBuffer& dest) {
}
return processed_count > 0;
}

void processMemoryMappingsFile(
Logger& logger,
std::string& file_path,
int64_t timestamp) {
std::ifstream mappingsFile(file_path);
if (!mappingsFile.is_open()) {
return;
}

int32_t tid = threadID();
std::string mappingLine;
while (std::getline(mappingsFile, mappingLine)) {
auto mappingId = logger.write(entries::StandardEntry{
.type = EntryType::MAPPING,
.tid = tid,
.timestamp = timestamp,
});
auto keyId = logger.writeBytes(
EntryType::STRING_KEY,
mappingId,
reinterpret_cast<const uint8_t*>(kMemoryMappingKey),
sizeof(kMemoryMappingKey));
logger.writeBytes(
EntryType::STRING_VALUE,
keyId,
reinterpret_cast<const uint8_t*>(mappingLine.c_str()),
mappingLine.size());
}
}

} // namespace

MmapBufferTraceWriter::MmapBufferTraceWriter(
Expand Down Expand Up @@ -234,11 +267,12 @@ int64_t MmapBufferTraceWriter::writeTrace(

auto entriesCount = mapBufferPrefix->header.size;
// Number of additional records we need to log in addition to entries from the
// buffer file + some buffer for long string entries.
constexpr auto kServiceRecordCount = 11 + 10 /* extra buffer */;
// buffer file + memory mappings file records + some buffer for long string
// entries.
constexpr auto kExtraRecordCount = 4096;

TraceBufferHolder bufferHolder =
TraceBuffer::allocate(entriesCount + kServiceRecordCount);
TraceBuffer::allocate(entriesCount + kExtraRecordCount);
TraceBuffer::Cursor startCursor = bufferHolder->currentHead();
Logger logger(
[&bufferHolder]() -> logger::PacketBuffer& { return *bufferHolder; });
Expand Down Expand Up @@ -287,6 +321,14 @@ int64_t MmapBufferTraceWriter::writeTrace(
std::string(mapBufferPrefix->header.sessionId));
loggerWriteQplTriggerAnnotation(
logger, qpl_marker_id, "type", type, timestamp);

const char* mapsFilename = mapBufferPrefix->header.memoryMapsFilename;
if (mapsFilename[0] != '\0') {
auto lastSlashIdx = dump_path.rfind("/");
std::string mapsPath = dump_path.substr(0, lastSlashIdx + 1) + mapsFilename;
processMemoryMappingsFile(logger, mapsPath, timestamp);
}

loggerWrite(logger, EntryType::TRACE_END, 0, 0, trace_id, timestamp);

TraceWriter writer(
Expand Down
110 changes: 70 additions & 40 deletions cpp/test/mmapbuf/writer/MmapBufferTraceWriterTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <zlib.h>
#include <zstr/zstr.hpp>
#include <climits>
#include <fstream>
#include <ostream>
#include <sstream>
#include <vector>
Expand Down Expand Up @@ -67,6 +68,11 @@ constexpr int64_t kTraceRecollectionTimestamp = 1234567;
constexpr int32_t kConfigId = 11;
constexpr int32_t kBuildId = 17;

static const std::array<std::string, 3> kMappings = {
"libhwui.so:722580c000:586015DEC7C4DA055D33796D9D793508:186000:491000",
"libart-compiler.so:71987dd000:25CFFF6256F96F117E72005B5318E262:c2000:244000",
"libc.so:7224896000:0965E88D999C749783C8947F9B7937E9:40000:a7000"};

#define EXPECT_STRING_CONTAINS(haystack, needle) \
EXPECT_PRED_FORMAT2(assertContainsInString, haystack, needle)

Expand All @@ -90,48 +96,17 @@ class MockCallbacks : public TraceCallbacks {
MOCK_METHOD2(onTraceAbort, void(int64_t, AbortReason));
};

struct StandardEntryWrapper {
StandardEntryWrapper(StandardEntry& standard_entry)
: entry(standard_entry), output(precomputeTraceOutput(entry)) {}

static std::string precomputeTraceOutput(
const StandardEntry& standard_entry) {
outstream().str("");
visitor().visit(standard_entry);
return outstream().str();
}

static std::stringstream& outstream() {
static std::stringstream ss{};
return ss;
}

static EntryVisitor& visitor() {
// Replicating part of visitor chain to get the expected output
static PrintEntryVisitor printVisitor(outstream());
static DeltaEncodingVisitor deltaVisitor(printVisitor);
static TimestampTruncatingVisitor timestampVisitor(deltaVisitor, 6);
return timestampVisitor;
}

StandardEntry entry;
std::string output;
};

class MmapBufferTraceWriterTest : public ::testing::Test {
protected:
MmapBufferTraceWriterTest()
: ::testing::Test(),
loggedEntries_(),
temp_dump_file_("test_dump"),
temp_trace_folder_(kTraceFolder) {
temp_trace_folder_(kTraceFolder),
temp_mappings_file_("maps", temp_dump_file_.path().parent_path()) {
std::srand(std::time(nullptr));
}

int dumpFd() {
return temp_dump_file_.fd();
}

const std::string& dumpPath() {
return temp_dump_file_.path().generic_string();
}
Expand Down Expand Up @@ -166,13 +141,22 @@ class MmapBufferTraceWriterTest : public ::testing::Test {
}

void writeRandomEntries(TraceBuffer& buf, int records_count) {
std::stringstream outstream{};
PrintEntryVisitor printVisitor(outstream);
DeltaEncodingVisitor deltaVisitor(printVisitor);
TimestampTruncatingVisitor visitor(deltaVisitor, 6);

Logger logger([&buf]() -> PacketBuffer& { return buf; });
// Write the main service entry before the main content
auto serviceEntry = generateTraceBackwardsEntry();
loggedEntries_.push_back(StandardEntryWrapper(serviceEntry));
outstream.str("");
visitor.visit(serviceEntry);
loggedEntries_.push_back(outstream.str());
while (records_count > 0) {
auto entry = generateRandomEntry();
loggedEntries_.push_back(StandardEntryWrapper(entry));
outstream.str("");
visitor.visit(entry);
loggedEntries_.push_back(outstream.str());
logger.write(std::move(entry));
--records_count;
}
Expand All @@ -182,13 +166,20 @@ class MmapBufferTraceWriterTest : public ::testing::Test {
writeTraceWithRandomEntries(records_count, records_count);
}

void writeTraceWithRandomEntries(int records_count, int buffer_size) {
void writeTraceWithRandomEntries(
int records_count,
int buffer_size,
bool set_mappings_file = false) {
MmapBufferManager bufManager{};
MmapBufferManagerTestAccessor bufManagerAccessor(bufManager);
bool res = bufManager.allocateBuffer(buffer_size, dumpPath(), 1, 1);
ASSERT_EQ(res, true) << "Unable to allocate the buffer";
bufManager.updateHeader(0, kQplId, kTraceId);
bufManager.updateId("272c3f80-f076-5a89-e265-60dcf407373b");
ASSERT_EQ(res, true) << "Unable to allocate the buffer";
if (set_mappings_file) {
bufManager.updateMemoryMappingFilename(
temp_mappings_file_.path().filename().generic_string());
}
TraceBufferHolder ringBuffer = TraceBuffer::allocateAt(
buffer_size, bufManagerAccessor.mmapBufferPointer());
writeRandomEntries(*ringBuffer, records_count);
Expand All @@ -198,6 +189,15 @@ class MmapBufferTraceWriterTest : public ::testing::Test {
<< "Unable to msync the buffer: " << strerror(errno);
}

void writeMemoryMappingsFile() {
// use output stream to write the content
std::ofstream mappingsFile(temp_mappings_file_.path().generic_string());
for (const std::string& map : kMappings) {
mappingsFile << map << std::endl;
}
mappingsFile.close();
}

fs::path getOnlyTraceFile() {
auto dir_iter = fs::recursive_directory_iterator(traceFolderPath());
auto is_file = [](const fs::directory_entry& x) {
Expand Down Expand Up @@ -227,16 +227,24 @@ class MmapBufferTraceWriterTest : public ::testing::Test {

void verifyLogEntriesFromTraceFile() {
auto traceContents = getOnlyTraceFileContents();
for (auto entry : loggedEntries_) {
for (std::string& entry : loggedEntries_) {
// Compare entry output ignoring first entry id (before "|")
auto entryOutput = entry.output.substr(entry.output.find("|"));
auto entryOutput = entry.substr(entry.find("|"));
EXPECT_STRING_CONTAINS(traceContents, entryOutput);
}
}

std::vector<StandardEntryWrapper> loggedEntries_;
void verifyMemoryMappingEntries() {
auto traceContents = getOnlyTraceFileContents();
for (const std::string& map : kMappings) {
EXPECT_STRING_CONTAINS(traceContents, map);
}
}

std::vector<std::string> loggedEntries_;
test::TemporaryFile temp_dump_file_;
test::TemporaryDirectory temp_trace_folder_;
test::TemporaryFile temp_mappings_file_;
};

TEST_F(MmapBufferTraceWriterTest, testDumpWriteAndRecollectEndToEnd) {
Expand All @@ -257,6 +265,28 @@ TEST_F(MmapBufferTraceWriterTest, testDumpWriteAndRecollectEndToEnd) {
verifyLogEntriesFromTraceFile();
}

TEST_F(
MmapBufferTraceWriterTest,
testDumpWriteAndRecollectEndToEndWithMappings) {
using ::testing::_;
writeTraceWithRandomEntries(10, 10, true);
writeMemoryMappingsFile();

std::string testFolder(traceFolderPath());
std::string testTracePrefix(kTracePrefix);
auto mockCallbacks = std::make_shared<::testing::NiceMock<MockCallbacks>>();
MmapBufferTraceWriter traceWriter(
testFolder, testTracePrefix, 0, mockCallbacks);

EXPECT_CALL(*mockCallbacks, onTraceStart(kTraceId, 0, _));
EXPECT_CALL(*mockCallbacks, onTraceEnd(kTraceId));

traceWriter.writeTrace(dumpPath(), "test", kTraceRecollectionTimestamp);

verifyLogEntriesFromTraceFile();
verifyMemoryMappingEntries();
}

TEST_F(
MmapBufferTraceWriterTest,
testAbortCallbackIsCalledWhenWriterThrowsException) {
Expand Down
40 changes: 11 additions & 29 deletions java/main/com/facebook/profilo/mmapbuf/MmapBufferFileHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,18 @@
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import javax.annotation.Nullable;

class MmapBufferFileHelper {

interface FileDeletionBlacklist {
List<String> getFilenames();
}

public static final String BUFFER_FILE_SUFFIX = ".buff";
public static final String MEMORY_MAPPING_FILE_SUFFIX = ".maps";

private static final int MAX_DUMPS_TO_KEEP = 3;

private static final Comparator<File> MOST_RECENT_FILES_COMPARATOR =
new Comparator<File>() {
@Override
public int compare(File f1, File f2) {
long f1LastModified = f1.lastModified();
long f2LastModified = f2.lastModified();
if (f1LastModified == f2LastModified) {
return 0;
}
if (f1LastModified < f2LastModified) {
return 1;
} else {
return -1;
}
}
};

private final File mMmapFilesFolder;
public static final Object DUMP_FILES_LOCK = new Object();

Expand Down Expand Up @@ -72,7 +56,7 @@ public boolean accept(File dir, String name) {
return foundFiles[0];
}

public void deleteOldBufferFiles() {
public void deleteOldBufferFiles(@Nullable FileDeletionBlacklist blacklist) {
File mmapFolder = getFolderIfExists();
if (mmapFolder == null) {
return;
Expand All @@ -81,13 +65,11 @@ public void deleteOldBufferFiles() {
if (mmapFiles == null) {
return;
}
int filesCount = mmapFiles.length;
if (filesCount <= MAX_DUMPS_TO_KEEP) {
return;
}
Arrays.sort(mmapFiles, MOST_RECENT_FILES_COMPARATOR);
for (int i = MAX_DUMPS_TO_KEEP; i < filesCount; i++) {
File file = mmapFiles[i];
List<String> blacklistFilenames = blacklist.getFilenames();
for (File file : mmapFiles) {
if (blacklistFilenames.contains(file.getName())) {
continue;
}
synchronized (DUMP_FILES_LOCK) {
if (file.exists()) {
file.delete();
Expand Down
15 changes: 15 additions & 0 deletions java/main/com/facebook/profilo/mmapbuf/MmapBufferManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import com.facebook.jni.annotations.DoNotStrip;
import com.facebook.soloader.SoLoader;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -60,6 +63,18 @@ public MmapBufferManager(long configId, File folder, Context context) {
return mMmapFileName;
}

public List<String> getCurrentFilenames() {
if (mMmapFileName == null) {
return Collections.emptyList();
}
ArrayList<String> filenames = new ArrayList<>(2);
filenames.add(mMmapFileName);
if (mMemoryMappingsFile != null) {
filenames.add(mMemoryMappingsFile.getName());
}
return filenames;
}

public boolean isEnabled() {
return mEnabled.get();
}
Expand Down

0 comments on commit c6e43d1

Please sign in to comment.