diff --git a/.github/ci/packages.apt b/.github/ci/packages.apt index c8ab7a2a..d94383c2 100644 --- a/.github/ci/packages.apt +++ b/.github/ci/packages.apt @@ -3,6 +3,7 @@ libcurl4-openssl-dev libgflags-dev libignition-cmake2-dev libignition-common4-dev +libignition-math6-dev libignition-msgs7-dev libignition-tools-dev libjsoncpp-dev diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d61cf13..90a36cfc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,11 @@ ign_find_package(ZIP REQUIRED PRIVATE) ign_find_package(ignition-common4 REQUIRED PRIVATE) set(IGN_COMMON_MAJOR_VER ${ignition-common4_VERSION_MAJOR}) +#-------------------------------------- +# Find ignition-math +ign_find_package(ignition-math6 REQUIRED PRIVATE) +set(IGN_MSGS_MAJOR_VER ${ignition-math6_VERSION_MAJOR}) + #-------------------------------------- # Find ignition-msgs ign_find_package(ignition-msgs7 REQUIRED PRIVATE) diff --git a/README.md b/README.md index 79fcc4fc..8c0f4df3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,10 @@ Test coverage reports are available at Codecov: # Building and installing -See the [installation tutorial](https://ignitionrobotics.org/api/fuel_tools/4.0/install.html). +See the [installation tutorial](https://ignitionrobotics.org/api/fuel_tools/5.0/install.html). + +Make sure `IGN_CONFIG_PATH` is set to the right install location so that `ign fuel` will work. +Default is `/usr/local/share/ignition`. ## Examples diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 880e0226..0ccba034 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: libignition-cmake2-dev libignition-common4-dev libignition-math6-dev - # libignition-msgs5-dev + libignition-msgs6-dev # Ignition tools - git clone http://github.com/ignitionrobotics/ign-tools -b master - cd ign-tools diff --git a/include/ignition/fuel_tools/ClientConfig.hh b/include/ignition/fuel_tools/ClientConfig.hh index 8f088385..b6522285 100644 --- a/include/ignition/fuel_tools/ClientConfig.hh +++ b/include/ignition/fuel_tools/ClientConfig.hh @@ -151,6 +151,10 @@ namespace ignition /// \return The list of servers. public: std::vector Servers() const; + /// \brief List of servers the client will connect to. + /// \return The list of servers. + public: std::vector & MutableServers() const; + /// \brief Add a server to the list. /// \param[in] _srv The server config. public: void AddServer(const ServerConfig &_srv); diff --git a/include/ignition/fuel_tools/FuelClient.hh b/include/ignition/fuel_tools/FuelClient.hh index b8207792..d49637ce 100644 --- a/include/ignition/fuel_tools/FuelClient.hh +++ b/include/ignition/fuel_tools/FuelClient.hh @@ -381,6 +381,13 @@ namespace ignition public: bool ParseCollectionUrl(const common::URI &_url, CollectionIdentifier &_id); + /// \brief Checked if there is any header already specify + /// \param[in] _serverConfig Server configuration + /// \param[inout] _headers Vector with headers to check + private: void AddServerConfigParametersToHeaders( + const ignition::fuel_tools::ServerConfig &_serverConfig, + std::vector &_headers) const; + /// \brief PIMPL private: std::unique_ptr dataPtr; }; diff --git a/src/ClientConfig.cc b/src/ClientConfig.cc index 28f7f7e6..f96d7de3 100644 --- a/src/ClientConfig.cc +++ b/src/ClientConfig.cc @@ -278,6 +278,7 @@ bool ClientConfig::LoadConfig(const std::string &_file) tokens.push("root"); std::string serverURL = ""; std::string cacheLocationConfig = ""; + std::string privateToken = ""; do { @@ -325,12 +326,18 @@ bool ClientConfig::LoadConfig(const std::string &_file) { // Sanity check: Make sure that the server is not already stored. bool repeated = false; - for (auto const savedServer : this->Servers()) + for (auto & savedServer : this->MutableServers()) { if (savedServer.Url().Str() == serverURL) { + if (!privateToken.empty()) + { + ignmsg << "Set private token for " << serverURL << " server." + << std::endl; + savedServer.SetApiKey(privateToken); + } ignwarn << "URL [" << serverURL << "] already exists. " - << "Ignoring server" << std::endl; + << "Ignoring server" << std::endl; repeated = true; break; } @@ -340,6 +347,12 @@ bool ClientConfig::LoadConfig(const std::string &_file) // Add the new server. ServerConfig newServer; newServer.SetUrl(common::URI(serverURL)); + if (!privateToken.empty()) + { + ignmsg << "Set private token for " << serverURL << " server." + << std::endl; + newServer.SetApiKey(privateToken); + } this->AddServer(newServer); } } @@ -372,6 +385,13 @@ bool ClientConfig::LoadConfig(const std::string &_file) cacheLocationConfig = path; tokens.pop(); } + else if (!tokens.empty() && tokens.top() == "private-token") + { + std::string token( + reinterpret_cast(event.data.scalar.value)); + privateToken = token; + tokens.pop(); + } else { std::string key( @@ -431,6 +451,12 @@ std::vector ClientConfig::Servers() const return this->dataPtr->servers; } +////////////////////////////////////////////////// +std::vector & ClientConfig::MutableServers() const +{ + return this->dataPtr->servers; +} + ////////////////////////////////////////////////// void ClientConfig::AddServer(const ServerConfig &_srv) { diff --git a/src/FuelClient.cc b/src/FuelClient.cc index 51b3511d..c3a3aa75 100644 --- a/src/FuelClient.cc +++ b/src/FuelClient.cc @@ -302,8 +302,11 @@ Result FuelClient::ModelDetails(const ModelIdentifier &_id, common::URIPath path; path = path / _id.Owner() / "models" / _id.Name(); + std::vector headersIncludingServerConfig = _headers; + AddServerConfigParametersToHeaders( + _id.Server(), headersIncludingServerConfig); resp = rest.Request(HttpMethod::GET, serverUrl, version, - path.Str(), {}, _headers, ""); + path.Str(), {}, headersIncludingServerConfig, ""); if (resp.statusCode != 200) return Result(ResultType::FETCH_ERROR); @@ -464,9 +467,13 @@ Result FuelClient::UploadModel(const std::string &_pathToModelDir, if (!this->dataPtr->FillModelForm(_pathToModelDir, _id, _private, form)) return Result(ResultType::UPLOAD_ERROR); + std::vector headersIncludingServerConfig = _headers; + AddServerConfigParametersToHeaders( + _id.Server(), headersIncludingServerConfig); // Send the request. resp = rest.Request(HttpMethod::POST_FORM, _id.Server().Url().Str(), - _id.Server().Version(), "models", {}, _headers, "", form); + _id.Server().Version(), "models", {}, + headersIncludingServerConfig, "", form); if (resp.statusCode != 200) { @@ -503,6 +510,27 @@ Result FuelClient::DeleteModel(const ModelIdentifier &) return Result(ResultType::DELETE_ERROR); } +void FuelClient::AddServerConfigParametersToHeaders( + const ignition::fuel_tools::ServerConfig &_serverConfig, + std::vector &_headers) const +{ + bool privateTokenDefined = false; + for (auto header : _headers) + { + if (header.find("Private-token:") != std::string::npos) + { + privateTokenDefined = true; + } + } + if (!privateTokenDefined) + { + if (!_serverConfig.ApiKey().empty()) + { + _headers.push_back("Private-token: " + _serverConfig.ApiKey()); + } + } +} + ////////////////////////////////////////////////// Result FuelClient::DeleteUrl(const ignition::common::URI &_uri, const std::vector &_headers) @@ -518,6 +546,7 @@ Result FuelClient::DeleteUrl(const ignition::common::URI &_uri, ModelIdentifier modelId; WorldIdentifier worldId; + std::vector headersIncludingServerConfig = _headers; if (this->ParseModelUrl(_uri, modelId)) { type = "model"; @@ -525,6 +554,8 @@ Result FuelClient::DeleteUrl(const ignition::common::URI &_uri, server = modelId.Server().Url().Str(); version = modelId.Server().Version(); path = path / modelId.Owner() / "models" / modelId.Name(); + AddServerConfigParametersToHeaders( + modelId.Server(), headersIncludingServerConfig); } else if (this->ParseWorldUrl(_uri, worldId)) { @@ -533,6 +564,8 @@ Result FuelClient::DeleteUrl(const ignition::common::URI &_uri, server = worldId.Server().Url().Str(); version = worldId.Server().Version(); path = path / worldId.Owner() / "worlds" / worldId.Name(); + AddServerConfigParametersToHeaders( + worldId.Server(), headersIncludingServerConfig); } else { @@ -542,7 +575,7 @@ Result FuelClient::DeleteUrl(const ignition::common::URI &_uri, // Send the request. resp = rest.Request(HttpMethod::DELETE, server, version, path.Str(), {}, - _headers, "", {}); + headersIncludingServerConfig, "", {}); if (resp.statusCode != 200) { @@ -586,11 +619,15 @@ Result FuelClient::DownloadModel(const ModelIdentifier &_id, ignmsg << "Downloading model [" << _id.UniqueName() << "]" << std::endl; + std::vector headersIncludingServerConfig = _headers; + AddServerConfigParametersToHeaders( + _id.Server(), headersIncludingServerConfig); // Request ignition::fuel_tools::Rest rest; RestResponse resp; resp = rest.Request(HttpMethod::GET, _id.Server().Url().Str(), - _id.Server().Version(), route.Str(), {}, _headers, ""); + _id.Server().Version(), route.Str(), {}, + headersIncludingServerConfig, ""); if (resp.statusCode != 200) { ignerr << "Failed to download model." << std::endl @@ -698,11 +735,16 @@ Result FuelClient::DownloadWorld(WorldIdentifier &_id) ignmsg << "Downloading world [" << _id.UniqueName() << "]" << std::endl; + std::vector headersIncludingServerConfig; + AddServerConfigParametersToHeaders( + _id.Server(), headersIncludingServerConfig); + // Request ignition::fuel_tools::Rest rest; RestResponse resp; resp = rest.Request(HttpMethod::GET, _id.Server().Url().Str(), - _id.Server().Version(), route.Str(), {}, {}, ""); + _id.Server().Version(), route.Str(), {}, + headersIncludingServerConfig, ""); if (resp.statusCode != 200) { ignerr << "Failed to download world." << std::endl @@ -1300,8 +1342,11 @@ Result FuelClient::PatchModel( form.emplace("private", _model.Private() ? "1" : "0"); } + std::vector headersIncludingServerConfig = _headers; + AddServerConfigParametersToHeaders( + _model.Server(), headersIncludingServerConfig); resp = rest.Request(HttpMethod::PATCH_FORM, serverUrl, version, - path.Str(), {}, _headers, "", form); + path.Str(), {}, headersIncludingServerConfig, "", form); if (resp.statusCode != 200) return Result(ResultType::PATCH_ERROR); diff --git a/src/LocalCache.cc b/src/LocalCache.cc index 5b3c85b2..31ccefa2 100644 --- a/src/LocalCache.cc +++ b/src/LocalCache.cc @@ -32,6 +32,7 @@ #include #include #include +#include #include "ignition/fuel_tools/ClientConfig.hh" #include "ignition/fuel_tools/Helpers.hh" @@ -462,11 +463,24 @@ bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir, // Get the element with the highest (most recent) version. tinyxml2::XMLElement *sdfElementLatest = nullptr; - double maxVersion = 0.0; + math::SemanticVersion maxVersion{"0.0"}; tinyxml2::XMLElement *sdfElement = modelElement->FirstChildElement("sdf"); while (sdfElement) { - double version = std::stod(sdfElement->Attribute("version")); + math::SemanticVersion version; + + auto versionAttribute = sdfElement->Attribute("version"); + if (nullptr == versionAttribute) + { + version.Parse("0.0.1"); + ignwarn << " element missing version attribute, assuming version [" + << version << "]" << std::endl; + } + else + { + version.Parse(versionAttribute); + } + if (version > maxVersion) { maxVersion = version; diff --git a/src/cmd/cmdfuel.rb.in b/src/cmd/cmdfuel.rb.in index df178d68..de00bef9 100755 --- a/src/cmd/cmdfuel.rb.in +++ b/src/cmd/cmdfuel.rb.in @@ -72,7 +72,10 @@ COMMANDS = { 'fuel' => " -v [ --verbose ] [arg] Adjust the level of console output (0~4). \n"\ " The default verbosity is 1, use -v without \n"\ " arguments for level 3. \n" + - COMMON_OPTIONS + COMMON_OPTIONS + "\n\n" + + "Environment variables: \n"\ + " IGN_FUEL_CACHE_PATH Path to the cache where resources are \n"\ + " downloaded to. Defaults to $HOME/.ignition/fuel \n" } SUBCOMMANDS = { diff --git a/tutorials/01_installation.md b/tutorials/01_installation.md index 2da8fe2f..991decd3 100644 --- a/tutorials/01_installation.md +++ b/tutorials/01_installation.md @@ -4,32 +4,30 @@ Next Tutorial: \ref configuration ## Overview -Instructions to install Ignition Fuel Tools on all the platforms -supported: major Linux distributions, Mac OS X and Windows. +Instructions to install Ignition Fuel Tools on all the platforms supported. -## Ubuntu Linux +## Binary Install + +### Ubuntu Linux Setup your computer to accept software from *packages.osrfoundation.org*: - -```{.sh} +``` sudo sh -c 'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-stable.list' ``` Setup keys: - ``` wget http://packages.osrfoundation.org/gazebo.key -O - | sudo apt-key add - ``` Install Ignition Fuel Tools: - ``` sudo apt-get update sudo apt-get install libignition-fuel-tools6-dev ``` -## Mac OS X +### Mac OS X Ignition Fuel Tools and several of its dependencies can be compiled on OS X with [Homebrew](http://brew.sh/) using the [osrf/simulation @@ -42,23 +40,39 @@ Here are the instructions: Install Homebrew, which should also prompt you to install the XCode command-line tools: - ``` ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` Run the following commands: - ``` brew tap osrf/simulation brew install ignition-fuel-tools6 ``` -## Windows +### Windows -At this moment, Windows instructions are not available. +Install [Conda package management system](https://docs.conda.io/projects/conda/en/latest/user-guide/install/download.html). +Miniconda suffices. + +Create if necessary, and activate a Conda environment: +``` +conda create -n ign-ws +conda activate ign-ws +``` + +Install: + +``` +conda install libignition-fuel-tools<#> --channel conda-forge +``` -## Install from sources (Ubuntu Linux) +Be sure to replace `<#>` with a number value, such as 1 or 2, depending on +which version you need. + +## Source Install + +### Ubuntu Linux For compiling the latest version of Ignition Fuel Tools you will need an Ubuntu distribution equal to 16.04 (Xenial) or newer. @@ -73,7 +87,7 @@ sudo apt-get remove libignition-fuel-tools6-dev Install prerequisites. A clean Ubuntu system will need: ``` -sudo apt-get install git cmake pkg-config python ruby-ronn libignition-cmake2-dev libignition-common4-dev libignition-msgs5-dev libignition-tools-dev libzip-dev libjsoncpp-dev libcurl4-openssl-dev libyaml-dev +sudo apt-get install git cmake pkg-config python ruby-ronn libignition-cmake2-dev libignition-common4-dev libignition-math6-dev libignition-msgs7-dev libignition-tools-dev libzip-dev libjsoncpp-dev libcurl4-openssl-dev libyaml-dev ``` Clone the repository into a directory and go into it: @@ -92,20 +106,21 @@ cd build Configure Ignition Fuel Tools (choose either method a or b below): -* **A**. Release mode: This will generate optimized code, but will not have debug symbols. Use this mode if you don't need to use GDB. -``` -cmake ../ -``` -Note: You can use a custom install path to make it easier to switch -between source and Debian installs: -``` -cmake -DCMAKE_INSTALL_PREFIX=/home/$USER/local ../ -``` +* A. Release mode: This will generate optimized code, but will not have debug symbols. Use this mode if you don't need to use GDB. + ``` + cmake ../ + ``` -* **B**. Debug mode: This will generate code with debug symbols. Ignition Fuel Tools will run slower, but you'll be able to use GDB. -``` -cmake -DCMAKE_BUILD_TYPE=Debug ../ -``` + Note: You can use a custom install path to make it easier to switch + between source and debian installs: + ``` + cmake -DCMAKE_INSTALL_PREFIX=/home/$USER/local ../ + ``` + +* B. Debug mode: This will generate code with debug symbols. Ignition Fuel Tools will run slower, but you'll be able to use GDB. + ``` + cmake -DCMAKE_BUILD_TYPE=Debug ../ + ``` The output from `cmake ../` may generate a number of errors and warnings about missing packages. You must install the missing packages that have @@ -115,38 +130,92 @@ in which you installed prerequisites). Make note of your install path, which is output from cmake and should look something like: - ``` -- Install prefix: /home/$USER/local ``` Build Ignition Fuel Tools: - ``` make -j4 ``` Install Ignition Fuel Tools: - ``` sudo make install ``` If you decide to install the library in a local directory you'll need to modify your `LD_LIBRARY_PATH`: - ``` echo "export LD_LIBRARY_PATH=/local/lib:$LD_LIBRARY_PATH" >> ~/.bashrc ``` -### Uninstalling Source-based Install +#### Uninstalling Source Install If you need to uninstall Ignition Fuel Tools or switch back to a Debian-based install when you currently have installed the library from source, navigate to your source code directory's build folders and run `make uninstall`: - -\code +``` cd /tmp/ign-fuel-tools/build sudo make uninstall -\endcode +``` + +### Windows + +#### Prerequisites + +First, follow the [ign-cmake](https://github.com/ignitionrobotics/ign-cmake) tutorial for installing Conda, Visual Studio, CMake, etc., prerequisites, and creating a Conda environment. + +Navigate to ``condabin`` if necessary to use the ``conda`` command (i.e., if Conda is not in your `PATH` environment variable. You can find the location of ``condabin`` in Anaconda Prompt, ``where conda``). + +Create if necessary, and activate a Conda environment: +``` +conda create -n ign-ws +conda activate ign-ws +``` + +Install dependencies: +``` +conda install jsoncpp libzip --channel conda-forge +``` + +Install Ignition dependencies: + +You can view available versions and their dependencies: +``` +conda search libignition-fuel-tools* --channel conda-forge --info +``` + +Install Ignition dependencies, replacing `<#>` with the desired versions: +``` +conda install libignition-cmake<#> libignition-common<#> libignition-msgs<#> libignition-tools<#> --channel conda-forge +``` + +#### Building from source + +1. Activate the Conda environment created in the prerequisites: + ``` + conda activate ign-ws + ``` + +2. Navigate to where you would like to build the library, and clone the repository. + ``` + # Optionally, append `-b ign-fuel-tools#` (replace # with a number) to check out a specific version + git clone https://github.com/ignitionrobotics/ign-fuel-tools.git + ``` + +3. Configure and build + ``` + cd ign-fuel-tools + mkdir build + cd build + cmake .. -DBUILD_TESTING=OFF # Optionally, -DCMAKE_INSTALL_PREFIX=path\to\install + cmake --build . --config Release + ``` + +4. Optionally, install. You wil likely need to run a terminal with admin privileges for this call to succeed. + ``` + cmake --install . --config Release + ``` + diff --git a/tutorials/02_configuration.md b/tutorials/02_configuration.md index d0c2097a..3a410a67 100644 --- a/tutorials/02_configuration.md +++ b/tutorials/02_configuration.md @@ -16,6 +16,7 @@ Ignition Fuel Tools accepts a YAML file with the following syntax: servers: - url: https://fuel.ignitionrobotics.org + private-token: # - # url: https://myserver @@ -27,6 +28,7 @@ servers: The `servers` section specifies all Fuel servers to interact with. For each server, you must specify the URL to send the HTTP requests. +If the server requires auth you can specify the token filling the optional field `private-token`. The `cache` section captures options related with the local storage of the assets. `path` specifies the local directory where all assets will be diff --git a/tutorials/03_command_line.md b/tutorials/03_command_line.md index b78321a3..4e938d73 100644 --- a/tutorials/03_command_line.md +++ b/tutorials/03_command_line.md @@ -88,6 +88,24 @@ Downloading world: Download succeeded. ``` +If the model is privated you can create a config file with your token. For example, create a file +`/tmp/my_config.yaml` with the following content and edit your token: + +```yaml +--- +# The list of servers. +servers: + - + url: https://fuel.ignitionrobotics.org + private-token: +``` + +Then try to download the model: + +```bash +ign fuel download -v 4 -u https://fuel.ignitionrobotics.org/1.0/OpenRobotics/worlds/Empty -c /tmp/my_config.yaml +``` + Worlds downloaded with the tool get conveniently organized into the same directory, which we call the "local cache". The path is broken down as follows: