diff --git a/AirLib/include/api/RpcLibClientBase.hpp b/AirLib/include/api/RpcLibClientBase.hpp index f0acd387cb..54c30d671a 100644 --- a/AirLib/include/api/RpcLibClientBase.hpp +++ b/AirLib/include/api/RpcLibClientBase.hpp @@ -51,7 +51,7 @@ class RpcLibClientBase { vector simListSceneObjects(const string& name_regex = string(".*")) const; Pose simGetObjectPose(const std::string& object_name) const; bool simSetObjectPose(const std::string& object_name, const Pose& pose, bool teleport = true); - + //task management APIs void cancelLastTask(const std::string& vehicle_name = ""); virtual RpcLibClientBase* waitOnLastTask(bool* task_result = nullptr, float timeout_sec = Utils::nan()); @@ -110,6 +110,8 @@ class RpcLibClientBase { void simSetBonePoses(const std::unordered_map& poses, const std::string& character_name = ""); std::unordered_map simGetBonePoses(const std::vector& bone_names, const std::string& character_name = "") const; + std::vector simSwapTextures(const std::string& tags, int tex_id = 0, int component_id = 0, int material_id = 0); + protected: void* getClient(); const void* getClient() const; diff --git a/AirLib/include/api/WorldSimApiBase.hpp b/AirLib/include/api/WorldSimApiBase.hpp index 949c5efd4a..4276682c03 100644 --- a/AirLib/include/api/WorldSimApiBase.hpp +++ b/AirLib/include/api/WorldSimApiBase.hpp @@ -46,6 +46,7 @@ class WorldSimApiBase { virtual Pose getObjectPose(const std::string& object_name) const = 0; virtual bool setObjectPose(const std::string& object_name, const Pose& pose, bool teleport) = 0; + virtual std::unique_ptr> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) = 0; //----------- APIs to control ACharacter in scene ----------/ virtual void charSetFaceExpression(const std::string& expression_name, float value, const std::string& character_name) = 0; virtual float charGetFaceExpression(const std::string& expression_name, const std::string& character_name) const = 0; diff --git a/AirLib/src/api/RpcLibClientBase.cpp b/AirLib/src/api/RpcLibClientBase.cpp index 82ba021c9b..a7b8dba397 100644 --- a/AirLib/src/api/RpcLibClientBase.cpp +++ b/AirLib/src/api/RpcLibClientBase.cpp @@ -278,6 +278,11 @@ vector RpcLibClientBase::simListSceneObjects(const string& name_regex) c return pimpl_->client.call("simListSceneObjects", name_regex).as>(); } +std::vector RpcLibClientBase::simSwapTextures(const std::string& tags, int tex_id, int component_id, int material_id) +{ + return pimpl_->client.call("simSwapTextures", tags, tex_id, component_id, material_id).as>(); +} + msr::airlib::Pose RpcLibClientBase::simGetObjectPose(const std::string& object_name) const { return pimpl_->client.call("simGetObjectPose", object_name).as().to(); diff --git a/AirLib/src/api/RpcLibServerBase.cpp b/AirLib/src/api/RpcLibServerBase.cpp index f4ae6b3edc..ffa06f197d 100644 --- a/AirLib/src/api/RpcLibServerBase.cpp +++ b/AirLib/src/api/RpcLibServerBase.cpp @@ -319,6 +319,10 @@ RpcLibServerBase::RpcLibServerBase(ApiProvider* api_provider, const std::string& return r; }); + pimpl_->server.bind("simSwapTextures", [&](const std::string tag, int tex_id, int component_id, int material_id) -> std::vector { + return *getWorldSimApi()->swapTextures(tag, tex_id, component_id, material_id); + }); + //if we don't suppress then server will bomb out for exceptions raised by any method pimpl_->server.suppress_exceptions(true); } diff --git a/PythonClient/airsim/client.py b/PythonClient/airsim/client.py index d3628d30ab..d2aebcec47 100644 --- a/PythonClient/airsim/client.py +++ b/PythonClient/airsim/client.py @@ -73,6 +73,9 @@ def confirmConnection(self): print(ver_info) print('') + def simSwapTextures(self, tags, tex_id = 0, component_id = 0, material_id = 0): + return self.client.call("simSwapTextures", tags, tex_id, component_id, material_id) + # time-of-day control def simSetTimeOfDay(self, is_enabled, start_datetime = "", is_start_datetime_dst = False, celestial_clock_speed = 1, update_interval_secs = 60, move_sun = True): return self.client.call('simSetTimeOfDay', is_enabled, start_datetime, is_start_datetime_dst, celestial_clock_speed, update_interval_secs, move_sun) diff --git a/Unreal/Plugins/AirSim/Content/StarterContent/Materials/TextureSwappableMaterial.uasset b/Unreal/Plugins/AirSim/Content/StarterContent/Materials/TextureSwappableMaterial.uasset new file mode 100644 index 0000000000..ac025e939a Binary files /dev/null and b/Unreal/Plugins/AirSim/Content/StarterContent/Materials/TextureSwappableMaterial.uasset differ diff --git a/Unreal/Plugins/AirSim/Source/TextureShuffleActor.cpp b/Unreal/Plugins/AirSim/Source/TextureShuffleActor.cpp new file mode 100644 index 0000000000..b7fd401dd1 --- /dev/null +++ b/Unreal/Plugins/AirSim/Source/TextureShuffleActor.cpp @@ -0,0 +1,33 @@ +#include "TextureShuffleActor.h" + +void ATextureShuffleActor::SwapTexture_Implementation(int tex_id, int component_id, int material_id) +{ + if (SwappableTextures.Num() < 1) + return; + + if (!MaterialCacheInitialized) + { + TArray components; + GetComponents(components); + NumComponents = components.Num(); + DynamicMaterialInstances.Init(nullptr, components[component_id]->GetNumMaterials()); + MaterialCacheInitialized = true; + } + + if (NumComponents == 0 || DynamicMaterialInstances.Num() == 0) + return; + + tex_id %= SwappableTextures.Num(); + component_id %= NumComponents; + material_id %= DynamicMaterialInstances.Num(); + + if (DynamicMaterialInstances[material_id] == nullptr) + { + DynamicMaterialInstances[material_id] = UMaterialInstanceDynamic::Create(DynamicMaterial, this); + TArray components; + GetComponents(components); + components[component_id]->SetMaterial(material_id, DynamicMaterialInstances[material_id]); + } + + DynamicMaterialInstances[material_id]->SetTextureParameterValue("TextureParameter", SwappableTextures[tex_id]); +} \ No newline at end of file diff --git a/Unreal/Plugins/AirSim/Source/TextureShuffleActor.h b/Unreal/Plugins/AirSim/Source/TextureShuffleActor.h new file mode 100644 index 0000000000..21d5cd8063 --- /dev/null +++ b/Unreal/Plugins/AirSim/Source/TextureShuffleActor.h @@ -0,0 +1,35 @@ +#pragma once + +#include "CoreMinimal.h" +#include "Materials/Material.h" +#include "common/common_utils/Utils.hpp" +#include "common/AirSimSettings.hpp" +#include "Engine/StaticMeshActor.h" +#include "TextureShuffleActor.generated.h" + + +UCLASS() +class AIRSIM_API ATextureShuffleActor : public AStaticMeshActor +{ + GENERATED_BODY() + +protected: + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = TextureShuffle) + UMaterialInterface *DynamicMaterial = nullptr; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = TextureShuffle) + TArray SwappableTextures; + +public: + + UFUNCTION(BlueprintNativeEvent) + void SwapTexture(int tex_id = 0, int component_id = 0, int material_id = 0); + +private: + bool MaterialCacheInitialized = false; + int NumComponents = -1; + + UPROPERTY() + TArray DynamicMaterialInstances; +}; \ No newline at end of file diff --git a/Unreal/Plugins/AirSim/Source/WorldSimApi.cpp b/Unreal/Plugins/AirSim/Source/WorldSimApi.cpp index 1627aaca44..5120219f46 100644 --- a/Unreal/Plugins/AirSim/Source/WorldSimApi.cpp +++ b/Unreal/Plugins/AirSim/Source/WorldSimApi.cpp @@ -1,5 +1,6 @@ #include "WorldSimApi.h" #include "AirBlueprintLib.h" +#include "TextureShuffleActor.h" #include "common/common_utils/Utils.hpp" #include "Weather/WeatherLib.h" @@ -116,6 +117,43 @@ void WorldSimApi::setWeatherParameter(WeatherParameter param, float val) UWeatherLib::setWeatherParamScalar(simmode_->GetWorld(), param_e, val); } +std::unique_ptr> WorldSimApi::swapTextures(const std::string& tag, int tex_id, int component_id, int material_id) +{ + auto swappedObjectNames = std::make_unique>(); + UAirBlueprintLib::RunCommandOnGameThread([this, &tag, tex_id, component_id, material_id, &swappedObjectNames]() { + //Split the tag string into individual tags. + TArray splitTags; + FString notSplit = FString(tag.c_str()); + FString next = ""; + while (notSplit.Split(",", &next, ¬Split)) + { + next.TrimStartInline(); + splitTags.Add(next); + } + notSplit.TrimStartInline(); + splitTags.Add(notSplit); + + //Texture swap on actors that have all of those tags. + TArray shuffleables; + UAirBlueprintLib::FindAllActor(simmode_, shuffleables); + for (auto *shuffler : shuffleables) + { + bool invalidChoice = false; + for (auto required_tag : splitTags) + { + invalidChoice |= !shuffler->ActorHasTag(FName(*required_tag)); + if (invalidChoice) + break; + } + + if (invalidChoice) + continue; + dynamic_cast(shuffler)->SwapTexture(tex_id, component_id, material_id); + swappedObjectNames->push_back(TCHAR_TO_UTF8(*shuffler->GetName())); + } + }, true); + return swappedObjectNames; +} //------------------------------------------------- Char APIs -----------------------------------------------------------/ diff --git a/Unreal/Plugins/AirSim/Source/WorldSimApi.h b/Unreal/Plugins/AirSim/Source/WorldSimApi.h index 66a87c5ae5..146426e275 100644 --- a/Unreal/Plugins/AirSim/Source/WorldSimApi.h +++ b/Unreal/Plugins/AirSim/Source/WorldSimApi.h @@ -31,6 +31,7 @@ class WorldSimApi : public msr::airlib::WorldSimApiBase { virtual void printLogMessage(const std::string& message, const std::string& message_param = "", unsigned char severity = 0) override; + virtual std::unique_ptr> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) override; virtual std::vector listSceneObjects(const std::string& name_regex) const override; virtual Pose getObjectPose(const std::string& object_name) const override; virtual bool setObjectPose(const std::string& object_name, const Pose& pose, bool teleport) override; diff --git a/UnrealPluginFiles.vcxproj b/UnrealPluginFiles.vcxproj index 9d812e2e29..e0a4bede41 100644 --- a/UnrealPluginFiles.vcxproj +++ b/UnrealPluginFiles.vcxproj @@ -93,6 +93,7 @@ + diff --git a/UnrealPluginFiles.vcxproj.filters b/UnrealPluginFiles.vcxproj.filters index 797cf97178..b0516486ad 100644 --- a/UnrealPluginFiles.vcxproj.filters +++ b/UnrealPluginFiles.vcxproj.filters @@ -108,6 +108,9 @@ Source Files + + Source Files + diff --git a/docs/apis.md b/docs/apis.md index 07d1ce99ba..5965d542b3 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -127,6 +127,7 @@ for response in responses: AirSim offers comprehensive images APIs to retrieve synchronized images from multiple cameras along with ground truth including depth, disparity, surface normals and vision. You can set the resolution, FOV, motion blur etc parameters in [settings.json](settings.md). There is also API for detecting collision state. See also [complete code](https://github.com/Microsoft/AirSim/tree/master/Examples/DataCollection/StereoImageGenerator.hpp) that generates specified number of stereo images and ground truth depth with normalization to camera plan, computation of disparity image and saving it to [pfm format](pfm.md). More on [image APIs and Computer Vision mode](image_apis.md). +For vision problems that can benefit from domain randomization, there is also an [object retexturing API](retexturing.md), which can be used in supported scenes. ### Pause and Continue APIs AirSim allows to pause and continue the simulation through `pause(is_paused)` API. To pause the simulation call `pause(True)` and to continue the simulation call `pause(False)`. You may have scenario, especially while using reinforcement learning, to run the simulation for specified amount of time and then automatically pause. While simulation is paused, you may then do some expensive computation, send a new command and then again run the simulation for specified amount of time. This can be achieved by API `continueForTime(seconds)`. This API runs the simulation for the specified number of seconds and then pauses the simulation. For example usage, please see [pause_continue_car.py](https://github.com/Microsoft/AirSim/tree/master/PythonClient//car/pause_continue_car.py) and [pause_continue_drone.py](https://github.com/Microsoft/AirSim/tree/master/PythonClient//multirotor/pause_continue_drone.py). diff --git a/docs/images/tex_shuffle_actor.png b/docs/images/tex_shuffle_actor.png new file mode 100644 index 0000000000..53e1136681 Binary files /dev/null and b/docs/images/tex_shuffle_actor.png differ diff --git a/docs/images/tex_swap_demo.gif b/docs/images/tex_swap_demo.gif new file mode 100644 index 0000000000..d9d18e0a0a Binary files /dev/null and b/docs/images/tex_swap_demo.gif differ diff --git a/docs/images/tex_swap_group_editing.png b/docs/images/tex_swap_group_editing.png new file mode 100644 index 0000000000..650d3962ac Binary files /dev/null and b/docs/images/tex_swap_group_editing.png differ diff --git a/docs/images/tex_swap_material.png b/docs/images/tex_swap_material.png new file mode 100644 index 0000000000..06f0289be9 Binary files /dev/null and b/docs/images/tex_swap_material.png differ diff --git a/docs/images/tex_swap_subset.png b/docs/images/tex_swap_subset.png new file mode 100644 index 0000000000..eec0eba388 Binary files /dev/null and b/docs/images/tex_swap_subset.png differ diff --git a/docs/retexturing.md b/docs/retexturing.md new file mode 100644 index 0000000000..7c24f02399 --- /dev/null +++ b/docs/retexturing.md @@ -0,0 +1,70 @@ +# Runtime Texture Swapping + +## How to Make An Actor Retexturable + +To be made texture-swappable, an actor must derive from the parent class TextureShuffleActor. +The parent class can be set via the settings tab in the actor's blueprint. + +![Parent Class](images/tex_shuffle_actor.png) + +After setting the parent class to TextureShuffActor, the object gains the member DynamicMaterial. +DynamicMaterial needs to be set--on all actor instances in the scene--to TextureSwappableMaterial. +Warning: Statically setting the Dynamic Material in the blueprint class may cause rendering errors. It seems to work better to set it on all the actor instances in the scene, using the details panel. + +![TextureSwappableMaterial](images/tex_swap_material.png) + +## How to Define the Set(s) of Textures to Choose From + +Typically, certain subsets of actors will share a set of texture options with each other. (e.g. walls that are part of the same building) + +It's easy to set up these groupings by using Unreal Engine's group editing functionality. +Select all the instances that should have the same texture selection, and add the textures to all of them simultaneously via the Details panel. +Use the same technique to add descriptive tags to groups of actors, which will be used to address them in the API. + +![Group Editing](images/tex_swap_group_editing.png) + +It's ideal to work from larger groupings to smaller groupings, simply deselecting actors to narrow down the grouping as you go, and applying any individual actor properties last. + +![Subset Editing](images/tex_swap_subset.png) + +## How to Swap Textures from the API + +The following API is available in C++ and python. (C++ shown) + +```C++ +std::vector simSwapTextures(const std::string& tags, int tex_id); +``` + +The string of "," or ", " delimited tags identifies on which actors to perform the swap. +The tex_id indexes the array of textures assigned to each actor undergoing a swap. +The function will return the list of objects which matched the provided tags and had the texture swap perfomed. +If tex_id is out-of-bounds for some object's texture set, it will be taken modulo the number of textures that were available. + +Demo (Python): + +```Python +import airsim +import time + +c = airsim.client.MultirotorClient() +print(c.simSwapTextures("furniture", 0)) +time.sleep(2) +print(c.simSwapTextures("chair", 1)) +time.sleep(2) +print(c.simSwapTextures("table", 1)) +time.sleep(2) +print(c.simSwapTextures("chair, right", 0)) +``` + +Results: + +```bash +['RetexturableChair', 'RetexturableChair2', 'RetexturableTable'] +['RetexturableChair', 'RetexturableChair2'] +['RetexturableTable'] +['RetexturableChair2'] +``` + +![Demo](images/tex_swap_demo.gif) + +Note that in this example, different textures were chosen on each actor for the same index value. \ No newline at end of file