Merge pull request microsoft#2296 from ironclownfish/texture_swap

Texture swap API

AirLib/include/api/RpcLibClientBase.hpp

@@ -51,7 +51,7 @@ class RpcLibClientBase {
     vector<string> simListSceneObjects(const string& name_regex = string(".*")) const;
     Pose simGetObjectPose(const std::string& object_name) const;
     bool simSetObjectPose(const std::string& object_name, const Pose& pose, bool teleport = true);
-    
+
     //task management APIs
     void cancelLastTask(const std::string& vehicle_name = "");
     virtual RpcLibClientBase* waitOnLastTask(bool* task_result = nullptr, float timeout_sec = Utils::nan<float>());
@@ -110,6 +110,8 @@ class RpcLibClientBase {
     void simSetBonePoses(const std::unordered_map<std::string, msr::airlib::Pose>& poses, const std::string& character_name = "");
     std::unordered_map<std::string, msr::airlib::Pose> simGetBonePoses(const std::vector<std::string>& bone_names, const std::string& character_name = "") const;
 
+	std::vector<std::string> simSwapTextures(const std::string& tags, int tex_id = 0, int component_id = 0, int material_id = 0);
+
 protected:
     void* getClient();
     const void* getClient() const;

AirLib/include/api/WorldSimApiBase.hpp

@@ -46,6 +46,7 @@ class WorldSimApiBase {
     virtual Pose getObjectPose(const std::string& object_name) const = 0;
     virtual bool setObjectPose(const std::string& object_name, const Pose& pose, bool teleport) = 0;
 
+	virtual std::unique_ptr<std::vector<std::string>> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) = 0;
     //----------- APIs to control ACharacter in scene ----------/
     virtual void charSetFaceExpression(const std::string& expression_name, float value, const std::string& character_name) = 0;
     virtual float charGetFaceExpression(const std::string& expression_name, const std::string& character_name) const = 0;

AirLib/src/api/RpcLibClientBase.cpp

@@ -278,6 +278,11 @@ vector<string> RpcLibClientBase::simListSceneObjects(const string& name_regex) c
     return pimpl_->client.call("simListSceneObjects", name_regex).as<vector<string>>();
 }
 
+std::vector<std::string> RpcLibClientBase::simSwapTextures(const std::string& tags, int tex_id, int component_id, int material_id)
+{
+	return pimpl_->client.call("simSwapTextures", tags, tex_id, component_id, material_id).as<vector<string>>();
+}
+
 msr::airlib::Pose RpcLibClientBase::simGetObjectPose(const std::string& object_name) const
 {
     return pimpl_->client.call("simGetObjectPose", object_name).as<RpcLibAdapatorsBase::Pose>().to();

AirLib/src/api/RpcLibServerBase.cpp

@@ -319,6 +319,10 @@ RpcLibServerBase::RpcLibServerBase(ApiProvider* api_provider, const std::string&
         return r;
     });
 
+	pimpl_->server.bind("simSwapTextures", [&](const std::string tag, int tex_id, int component_id, int material_id) -> std::vector<string> {
+		return *getWorldSimApi()->swapTextures(tag, tex_id, component_id, material_id);
+	});
+
     //if we don't suppress then server will bomb out for exceptions raised by any method
     pimpl_->server.suppress_exceptions(true);
 }

PythonClient/airsim/client.py

@@ -73,6 +73,9 @@ def confirmConnection(self):
             print(ver_info)
         print('')
 
+    def simSwapTextures(self, tags, tex_id = 0, component_id = 0, material_id = 0):
+        return self.client.call("simSwapTextures", tags, tex_id, component_id, material_id)
+
     # time-of-day control
     def simSetTimeOfDay(self, is_enabled, start_datetime = "", is_start_datetime_dst = False, celestial_clock_speed = 1, update_interval_secs = 60, move_sun = True):
         return self.client.call('simSetTimeOfDay', is_enabled, start_datetime, is_start_datetime_dst, celestial_clock_speed, update_interval_secs, move_sun)

Unreal/Plugins/AirSim/Source/TextureShuffleActor.cpp

@@ -0,0 +1,33 @@
+#include "TextureShuffleActor.h"
+
+void ATextureShuffleActor::SwapTexture_Implementation(int tex_id, int component_id, int material_id)
+{
+	if (SwappableTextures.Num() < 1)
+		return;
+
+	if (!MaterialCacheInitialized)
+	{
+		TArray<UStaticMeshComponent*> components;
+		GetComponents<UStaticMeshComponent>(components);
+		NumComponents = components.Num();
+		DynamicMaterialInstances.Init(nullptr, components[component_id]->GetNumMaterials());
+		MaterialCacheInitialized = true;
+	}
+
+	if (NumComponents == 0 || DynamicMaterialInstances.Num() == 0)
+		return;
+
+	tex_id %= SwappableTextures.Num();
+	component_id %= NumComponents;
+	material_id %= DynamicMaterialInstances.Num();
+
+	if (DynamicMaterialInstances[material_id] == nullptr)
+	{
+		DynamicMaterialInstances[material_id] = UMaterialInstanceDynamic::Create(DynamicMaterial, this);
+		TArray<UStaticMeshComponent*> components;
+		GetComponents<UStaticMeshComponent>(components);
+		components[component_id]->SetMaterial(material_id, DynamicMaterialInstances[material_id]);
+	}
+
+	DynamicMaterialInstances[material_id]->SetTextureParameterValue("TextureParameter", SwappableTextures[tex_id]);
+}

Unreal/Plugins/AirSim/Source/TextureShuffleActor.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Materials/Material.h"
+#include "common/common_utils/Utils.hpp"
+#include "common/AirSimSettings.hpp"
+#include "Engine/StaticMeshActor.h"
+#include "TextureShuffleActor.generated.h"
+
+
+UCLASS()
+class AIRSIM_API ATextureShuffleActor : public AStaticMeshActor
+{
+	GENERATED_BODY()
+
+protected:
+
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = TextureShuffle)
+	UMaterialInterface *DynamicMaterial = nullptr;
+
+	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = TextureShuffle)
+	TArray<UTexture2D*> SwappableTextures;
+
+public:
+
+	UFUNCTION(BlueprintNativeEvent)
+	void SwapTexture(int tex_id = 0, int component_id = 0, int material_id = 0);
+
+private:
+	bool MaterialCacheInitialized = false;
+	int NumComponents = -1;
+
+	UPROPERTY()
+	TArray<UMaterialInstanceDynamic*> DynamicMaterialInstances;
+};

Unreal/Plugins/AirSim/Source/WorldSimApi.cpp

@@ -1,5 +1,6 @@
 #include "WorldSimApi.h"
 #include "AirBlueprintLib.h"
+#include "TextureShuffleActor.h"
 #include "common/common_utils/Utils.hpp"
 #include "Weather/WeatherLib.h"
 
@@ -116,6 +117,43 @@ void WorldSimApi::setWeatherParameter(WeatherParameter param, float val)
     UWeatherLib::setWeatherParamScalar(simmode_->GetWorld(), param_e, val);
 }
 
+std::unique_ptr<std::vector<std::string>> WorldSimApi::swapTextures(const std::string& tag, int tex_id, int component_id, int material_id)
+{
+	auto swappedObjectNames = std::make_unique<std::vector<std::string>>();
+	UAirBlueprintLib::RunCommandOnGameThread([this, &tag, tex_id, component_id, material_id, &swappedObjectNames]() {
+		//Split the tag string into individual tags.
+		TArray<FString> splitTags;
+		FString notSplit = FString(tag.c_str());
+		FString next = "";
+		while (notSplit.Split(",", &next, &notSplit))
+		{
+			next.TrimStartInline();
+			splitTags.Add(next);
+		}
+		notSplit.TrimStartInline();
+		splitTags.Add(notSplit);
+
+		//Texture swap on actors that have all of those tags.
+		TArray<AActor*> shuffleables;
+		UAirBlueprintLib::FindAllActor<ATextureShuffleActor>(simmode_, shuffleables);
+		for (auto *shuffler : shuffleables)
+		{
+			bool invalidChoice = false;
+			for (auto required_tag : splitTags)
+			{
+				invalidChoice |= !shuffler->ActorHasTag(FName(*required_tag));
+				if (invalidChoice)
+					break;
+			}
+
+			if (invalidChoice)
+				continue;
+			dynamic_cast<ATextureShuffleActor*>(shuffler)->SwapTexture(tex_id, component_id, material_id);
+			swappedObjectNames->push_back(TCHAR_TO_UTF8(*shuffler->GetName()));
+		}
+	}, true);
+	return swappedObjectNames;
+}
 
 //------------------------------------------------- Char APIs -----------------------------------------------------------/
 

Unreal/Plugins/AirSim/Source/WorldSimApi.h

@@ -31,6 +31,7 @@ class WorldSimApi : public msr::airlib::WorldSimApiBase {
     virtual void printLogMessage(const std::string& message,
         const std::string& message_param = "", unsigned char severity = 0) override;
 
+	virtual std::unique_ptr<std::vector<std::string>> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) override;
     virtual std::vector<std::string> listSceneObjects(const std::string& name_regex) const override;
     virtual Pose getObjectPose(const std::string& object_name) const override;
     virtual bool setObjectPose(const std::string& object_name, const Pose& pose, bool teleport) override;

UnrealPluginFiles.vcxproj

@@ -93,6 +93,7 @@
     <ClCompile Include="unreal\plugins\airsim\source\simjoystick\SimJoyStick.cpp" />
     <ClCompile Include="unreal\plugins\airsim\source\simmode\SimModeBase.cpp" />
     <ClCompile Include="unreal\plugins\airsim\source\simmode\SimModeWorldBase.cpp" />
+    <ClCompile Include="Unreal\Plugins\AirSim\Source\TextureShuffleActor.cpp" />
     <ClCompile Include="Unreal\Plugins\AirSim\Source\UnrealImageCapture.cpp" />
     <ClCompile Include="Unreal\Plugins\AirSim\Source\UnrealSensors\UnrealDistanceSensor.cpp" />
     <ClCompile Include="Unreal\Plugins\AirSim\Source\UnrealSensors\UnrealLidarSensor.cpp" />

UnrealPluginFiles.vcxproj.filters

@@ -108,6 +108,9 @@
     <ClCompile Include="Unreal\Plugins\AirSim\Source\UnrealSensors\UnrealLidarSensor.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="Unreal\Plugins\AirSim\Source\TextureShuffleActor.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="unreal\plugins\airsim\source\AirBlueprintLib.h">

docs/apis.md

@@ -127,6 +127,7 @@ for response in responses:
 AirSim offers comprehensive images APIs to retrieve synchronized images from multiple cameras along with ground truth including depth, disparity, surface normals and vision. You can set the resolution, FOV, motion blur etc parameters in [settings.json](settings.md). There is also API for detecting collision state. See also [complete code](https://github.com/Microsoft/AirSim/tree/master/Examples/DataCollection/StereoImageGenerator.hpp) that generates specified number of stereo images and ground truth depth with normalization to camera plan, computation of disparity image and saving it to [pfm format](pfm.md).
 
 More on [image APIs and Computer Vision mode](image_apis.md).
+For vision problems that can benefit from domain randomization, there is also an [object retexturing API](retexturing.md), which can be used in supported scenes.
 
 ### Pause and Continue APIs
 AirSim allows to pause and continue the simulation through `pause(is_paused)` API. To pause the simulation call `pause(True)` and to continue the simulation call `pause(False)`. You may have scenario, especially while using reinforcement learning, to run the simulation for specified amount of time and then automatically pause. While simulation is paused, you may then do some expensive computation, send a new command and then again run the simulation for specified amount of time. This can be achieved by API `continueForTime(seconds)`. This API runs the simulation for the specified number of seconds and then pauses the simulation. For example usage, please see [pause_continue_car.py](https://github.com/Microsoft/AirSim/tree/master/PythonClient//car/pause_continue_car.py) and [pause_continue_drone.py](https://github.com/Microsoft/AirSim/tree/master/PythonClient//multirotor/pause_continue_drone.py).

docs/retexturing.md

@@ -0,0 +1,70 @@
+# Runtime Texture Swapping
+
+## How to Make An Actor Retexturable
+
+To be made texture-swappable, an actor must derive from the parent class TextureShuffleActor.
+The parent class can be set via the settings tab in the actor's blueprint.
+
+![Parent Class](images/tex_shuffle_actor.png)
+
+After setting the parent class to TextureShuffActor, the object gains the member DynamicMaterial.
+DynamicMaterial needs to be set--on all actor instances in the scene--to TextureSwappableMaterial.
+Warning: Statically setting the Dynamic Material in the blueprint class may cause rendering errors. It seems to work better to set it on all the actor instances in the scene, using the details panel.
+
+![TextureSwappableMaterial](images/tex_swap_material.png)
+
+## How to Define the Set(s) of Textures to Choose From
+
+Typically, certain subsets of actors will share a set of texture options with each other. (e.g. walls that are part of the same building)
+
+It's easy to set up these groupings by using Unreal Engine's group editing functionality.
+Select all the instances that should have the same texture selection, and add the textures to all of them simultaneously via the Details panel.
+Use the same technique to add descriptive tags to groups of actors, which will be used to address them in the API.
+
+![Group Editing](images/tex_swap_group_editing.png)
+
+It's ideal to work from larger groupings to smaller groupings, simply deselecting actors to narrow down the grouping as you go, and applying any individual actor properties last.
+
+![Subset Editing](images/tex_swap_subset.png)
+
+## How to Swap Textures from the API
+
+The following API is available in C++ and python. (C++ shown)
+
+```C++
+std::vector<std::string> simSwapTextures(const std::string& tags, int tex_id);
+```
+
+The string of "," or ", " delimited tags identifies on which actors to perform the swap.
+The tex_id indexes the array of textures assigned to each actor undergoing a swap.
+The function will return the list of objects which matched the provided tags and had the texture swap perfomed.
+If tex_id is out-of-bounds for some object's texture set, it will be taken modulo the number of textures that were available.
+
+Demo (Python):
+
+```Python
+import airsim
+import time
+
+c = airsim.client.MultirotorClient()
+print(c.simSwapTextures("furniture", 0))
+time.sleep(2)
+print(c.simSwapTextures("chair", 1))
+time.sleep(2)
+print(c.simSwapTextures("table", 1))
+time.sleep(2)
+print(c.simSwapTextures("chair, right", 0))
+```
+
+Results:
+
+```bash
+['RetexturableChair', 'RetexturableChair2', 'RetexturableTable']
+['RetexturableChair', 'RetexturableChair2']
+['RetexturableTable']
+['RetexturableChair2']
+```
+
+![Demo](images/tex_swap_demo.gif)
+
+Note that in this example, different textures were chosen on each actor for the same index value.