Skip to content
This repository has been archived by the owner on Aug 8, 2023. It is now read-only.

Commit

Permalink
[test-runner] Add render test runner options for injecting probes
Browse files Browse the repository at this point in the history 
This allows injection of memory, gfx and network probes before and
after operations defined in render test are run.
  • Loading branch information
@alexshalamov
alexshalamov committed Nov 22, 2019
1 parent a02ed76 commit 17a5441
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 48 deletions.
24 changes: 20 additions & 4 deletions render-test/render_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,14 @@ void operator delete(void* ptr, size_t) noexcept {
#endif

namespace {
using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
using ArgumentsTuple =
std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string, std::set<std::string>>;
ArgumentsTuple parseArguments(int argc, char** argv) {
args::ArgumentParser argumentParser("Mapbox GL Test Runner");

static const std::unordered_map<std::string, std::string> probeMap{
{"memory", "probeMemory"}, {"network", "probeNetwork"}, {"gfx", "probeGFX"}};

args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});

args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
Expand All @@ -53,6 +57,12 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
args::MapFlagList<std::string, std::string> probes(
argumentParser,
"probe",
"Probe to be injected into a test. Supported values are: [memory, gfx, network]",
{"probe"},
probeMap);

try {
argumentParser.ParseCLI(argc, argv);
Expand Down Expand Up @@ -86,6 +96,9 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
exit(4);
}

const auto& probeValues = args::get(probes);
std::set<std::string> injectedProbes(probeValues.begin(), probeValues.end());

auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
Expand All @@ -95,7 +108,8 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
seed,
manifestPath.string(),
std::move(testNames),
std::move(testFilter)};
std::move(testFilter),
std::move(injectedProbes)};
}
} // namespace
namespace mbgl {
Expand All @@ -107,8 +121,10 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
std::string manifestPath;
std::vector<std::string> testNames;
std::string testFilter;
std::set<std::string> injectedProbes;

std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter, injectedProbes) =
parseArguments(argc, argv);
auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
if (!manifestData) {
exit(5);
Expand Down Expand Up @@ -159,7 +175,7 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {

bool errored = !metadata.errorMessage.empty();
if (!errored) {
errored = !runner.run(metadata) || !metadata.errorMessage.empty();
errored = !runner.run(metadata, injectedProbes) || !metadata.errorMessage.empty();
}

bool passed =
Expand Down
196 changes: 153 additions & 43 deletions render-test/runner.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,42 @@

using namespace mbgl;

namespace {
static const std::string waitOp("wait");
static const std::string sleepOp("sleep");
static const std::string addImageOp("addImage");
static const std::string updateImageOp("updateImage");
static const std::string removeImageOp("removeImage");
static const std::string setStyleOp("setStyle");
static const std::string setCenterOp("setCenter");
static const std::string setZoomOp("setZoom");
static const std::string setBearingOp("setBearing");
static const std::string setPitchOp("setPitch");
static const std::string setFilterOp("setFilter");
static const std::string setLayerZoomRangeOp("setLayerZoomRange");
static const std::string setLightOp("setLight");
static const std::string addLayerOp("addLayer");
static const std::string removeLayerOp("removeLayer");
static const std::string addSourceOp("addSource");
static const std::string removeSourceOp("removeSource");
static const std::string setPaintPropertyOp("setPaintProperty");
static const std::string setLayoutPropertyOp("setLayoutProperty");
static const std::string fileSizeProbeOp("probeFileSize");
static const std::string memoryProbeOp("probeMemory");
static const std::string memoryProbeStartOp("probeMemoryStart");
static const std::string memoryProbeEndOp("probeMemoryEnd");
static const std::string networkProbeOp("probeNetwork");
static const std::string networkProbeStartOp("probeNetworkStart");
static const std::string networkProbeEndOp("probeNetworkEnd");
static const std::string setFeatureStateOp("setFeatureState");
static const std::string getFeatureStateOp("getFeatureState");
static const std::string removeFeatureStateOp("removeFeatureState");
static const std::string panGestureOp("panGesture");
static const std::string gfxProbeOp("probeGFX");
static const std::string gfxProbeStartOp("probeGFXStart");
static const std::string gfxProbeEndOp("probeGFXEnd");
} // namespace

GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
: numBuffers(stats.numBuffers),
numDrawCalls(stats.numDrawCalls),
Expand Down Expand Up @@ -514,40 +550,6 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, R
auto& map = maps[key]->map;
auto& observer = maps[key]->observer;

static const std::string waitOp("wait");
static const std::string sleepOp("sleep");
static const std::string addImageOp("addImage");
static const std::string updateImageOp("updateImage");
static const std::string removeImageOp("removeImage");
static const std::string setStyleOp("setStyle");
static const std::string setCenterOp("setCenter");
static const std::string setZoomOp("setZoom");
static const std::string setBearingOp("setBearing");
static const std::string setPitchOp("setPitch");
static const std::string setFilterOp("setFilter");
static const std::string setLayerZoomRangeOp("setLayerZoomRange");
static const std::string setLightOp("setLight");
static const std::string addLayerOp("addLayer");
static const std::string removeLayerOp("removeLayer");
static const std::string addSourceOp("addSource");
static const std::string removeSourceOp("removeSource");
static const std::string setPaintPropertyOp("setPaintProperty");
static const std::string setLayoutPropertyOp("setLayoutProperty");
static const std::string fileSizeProbeOp("probeFileSize");
static const std::string memoryProbeOp("probeMemory");
static const std::string memoryProbeStartOp("probeMemoryStart");
static const std::string memoryProbeEndOp("probeMemoryEnd");
static const std::string networkProbeOp("probeNetwork");
static const std::string networkProbeStartOp("probeNetworkStart");
static const std::string networkProbeEndOp("probeNetworkEnd");
static const std::string setFeatureStateOp("setFeatureState");
static const std::string getFeatureStateOp("getFeatureState");
static const std::string removeFeatureStateOp("removeFeatureState");
static const std::string panGestureOp("panGesture");
static const std::string gfxProbeOp("probeGFX");
static const std::string gfxProbeStartOp("probeGFXStart");
static const std::string gfxProbeEndOp("probeGFXEnd");

if (operationArray[0].GetString() == waitOp) {
// wait
try {
Expand Down Expand Up @@ -1114,7 +1116,7 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)

TestRunner::Impl::~Impl() {}

bool TestRunner::run(TestMetadata& metadata) {
bool TestRunner::run(TestMetadata& metadata, const std::set<std::string>& injectedProbes) {
AllocationIndex::setActive(false);
AllocationIndex::reset();
ProxyFileSource::setTrackingActive(false);
Expand All @@ -1129,6 +1131,13 @@ bool TestRunner::run(TestMetadata& metadata) {
auto& frontend = maps[key]->frontend;
auto& map = maps[key]->map;

RunContext ctx{};

// Run 'begin' probes provided via command line arguments.
if (!runInjectedProbesBegin(metadata, injectedProbes, ctx)) {
return false;
}

frontend.setSize(metadata.size);
map.setSize(metadata.size);

Expand All @@ -1138,21 +1147,22 @@ bool TestRunner::run(TestMetadata& metadata) {
map.getStyle().loadJSON(serializeJsonValue(metadata.document));
map.jumpTo(map.getStyle().getDefaultCamera());

RunContext ctx{};
if (!runOperations(key, metadata, ctx)) return false;

if (!runOperations(key, metadata, ctx)) {
HeadlessFrontend::RenderResult result;
try {
if (metadata.outputsImage) result = frontend.render(map);
} catch (const std::exception&) {
return false;
}

mbgl::PremultipliedImage image;
try {
if (metadata.outputsImage) image = frontend.render(map).image;
} catch (const std::exception&) {
// Run 'end' probes provided via command line arguments
if (!runInjectedProbesEnd(metadata, injectedProbes, ctx, result.stats)) {
return false;
}

if (metadata.renderTest) {
return checkRenderTestResults(std::move(image), metadata);
return checkRenderTestResults(std::move(result.image), metadata);
} else {
std::vector<mbgl::Feature> features;
assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray());
Expand All @@ -1162,8 +1172,108 @@ bool TestRunner::run(TestMetadata& metadata) {
} else {
features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions);
}
return checkQueryTestResults(std::move(image), std::move(features), metadata);
return checkQueryTestResults(std::move(result.image), std::move(features), metadata);
}
}

using InjectedProbeMap = std::map<std::string, std::function<void(TestMetadata&, RunContext&)>>;
bool runInjectedProbe(TestMetadata& metadata,
const std::set<std::string>& probes,
RunContext& ctx,
const InjectedProbeMap& probeMap) {
for (const auto& probe : probes) {
auto it = probeMap.find(probe);
if (it == probeMap.end()) {
metadata.errorMessage = std::string("Unsupported operation: ") + probe;
return false;
}
it->second(metadata, ctx);
}
return true;
}

bool TestRunner::runInjectedProbesBegin(TestMetadata& metadata, const std::set<std::string>& probes, RunContext& ctx) {
const std::string mark = " - default - start";
static const InjectedProbeMap beginInjectedProbeMap = {
{// Injected memory probe begin
memoryProbeOp,
[&mark](TestMetadata& metadata, RunContext&) {
assert(!AllocationIndex::isActive());
AllocationIndex::setActive(true);
metadata.metrics.memory.emplace(std::piecewise_construct,
std::forward_as_tuple(memoryProbeOp + mark),
std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
AllocationIndex::getAllocationsCount()));
}},
{// Injected gfx probe begin
gfxProbeOp,
[](TestMetadata&, RunContext& ctx) {
assert(!ctx.gfxProbeActive);
ctx.gfxProbeActive = true;
ctx.baselineGfxProbe = ctx.activeGfxProbe;
}},
{// Injected network probe begin
networkProbeOp,
[&mark](TestMetadata& metadata, RunContext&) {
assert(!ProxyFileSource::isTrackingActive());
ProxyFileSource::setTrackingActive(true);
metadata.metrics.network.emplace(
std::piecewise_construct,
std::forward_as_tuple(networkProbeOp + mark),
std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
}}};

return runInjectedProbe(metadata, probes, ctx, beginInjectedProbeMap);
}

bool TestRunner::runInjectedProbesEnd(TestMetadata& metadata,
const std::set<std::string>& probes,
RunContext& ctx,
mbgl::gfx::RenderingStats stats) {
const std::string mark = " - default - end";
static const InjectedProbeMap endInjectedProbeMap = {
{// Injected memory probe end
memoryProbeOp,
[&mark](TestMetadata& metadata, RunContext&) {
assert(AllocationIndex::isActive());
auto emplaced =
metadata.metrics.memory.emplace(std::piecewise_construct,
std::forward_as_tuple(memoryProbeOp + mark),
std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
AllocationIndex::getAllocationsCount()));
assert(emplaced.second);
// TODO: Improve tolerance handling for memory tests.
emplaced.first->second.tolerance = 0.2f;
AllocationIndex::setActive(false);
AllocationIndex::reset();
}},
{// Injected gfx probe end
gfxProbeOp,
[&mark, &stats](TestMetadata& metadata, RunContext& ctx) {
assert(ctx.gfxProbeActive);
ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe);

// Compare memory allocations to the baseline probe
GfxProbe metricProbe = ctx.activeGfxProbe;
metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
metadata.metrics.gfx.insert({gfxProbeOp + mark, metricProbe});

ctx.gfxProbeActive = false;
}},
{// Injected network probe end
networkProbeOp,
[&mark](TestMetadata& metadata, RunContext&) {
assert(ProxyFileSource::isTrackingActive());
metadata.metrics.network.emplace(
std::piecewise_construct,
std::forward_as_tuple(networkProbeOp + mark),
std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
ProxyFileSource::setTrackingActive(false);
}}};

return runInjectedProbe(metadata, probes, ctx, endInjectedProbeMap);
}

void TestRunner::reset() {
Expand Down
5 changes: 4 additions & 1 deletion render-test/runner.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ struct TestMetadata;
class TestRunner {
public:
explicit TestRunner(Manifest);
bool run(TestMetadata&);
bool run(TestMetadata&, const std::set<std::string>&);
void reset();

// Manifest
Expand All @@ -24,6 +24,9 @@ class TestRunner {

private:
bool runOperations(const std::string& key, TestMetadata&, RunContext&);
bool runInjectedProbesBegin(TestMetadata&, const std::set<std::string>&, RunContext&);
bool runInjectedProbesEnd(TestMetadata&, const std::set<std::string>&, RunContext&, mbgl::gfx::RenderingStats);

bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
std::vector<mbgl::Feature>&& features,
TestMetadata&);
Expand Down

0 comments on commit 17a5441

Please sign in to comment.