From 01030930006c92942c6f31b2fba534f8d15c3cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20De=20Lillo?= Date: Thu, 31 Jan 2019 18:46:37 +0100 Subject: [PATCH 1/6] [software] Add new software `LightingEstimation` --- src/software/utils/CMakeLists.txt | 14 ++ .../utils/main_lightingEstimation.cpp | 134 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 src/software/utils/main_lightingEstimation.cpp diff --git a/src/software/utils/CMakeLists.txt b/src/software/utils/CMakeLists.txt index d1e0f8c539..5606eaee32 100644 --- a/src/software/utils/CMakeLists.txt +++ b/src/software/utils/CMakeLists.txt @@ -221,4 +221,18 @@ alicevision_add_software(aliceVision_utils_fisheyeProjection ${Boost_LIBRARIES} ) +# Lighting estimation from picture, albedo and geometry +alicevision_add_software(aliceVision_utils_lightingEstimation + SOURCE main_lightingEstimation.cpp + FOLDER ${FOLDER_SOFTWARE_UTILS} + LINKS aliceVision_system + aliceVision_numeric + aliceVision_sfmData + aliceVision_sfmDataIO + aliceVision_mvsUtils + aliceVision_image + aliceVision_lightingEstimation + ${Boost_LIBRARIES} +) + endif() # ALICEVISION_BUILD_SFM diff --git a/src/software/utils/main_lightingEstimation.cpp b/src/software/utils/main_lightingEstimation.cpp new file mode 100644 index 0000000000..f1ce479a07 --- /dev/null +++ b/src/software/utils/main_lightingEstimation.cpp @@ -0,0 +1,134 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2017 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +// These constants define the current software version. +// They must be updated when the command line is changed. +#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 +#define ALICEVISION_SOFTWARE_VERSION_MINOR 0 + +using namespace aliceVision; + +namespace po = boost::program_options; +namespace fs = boost::filesystem; + +int main(int argc, char** argv) +{ + system::Timer timer; + + // command-line parameters + + std::string verboseLevel = aliceVision::system::EVerboseLevel_enumToString(aliceVision::system::Logger::getDefaultVerboseLevel()); + std::string sfmDataFilename; + std::string depthMapsFilterFolder; + std::string imagesFolder; + std::string outputFolder; + + po::options_description allParams("AliceVision lighthingEstimation"); + + po::options_description requiredParams("Required parameters"); + requiredParams.add_options() + ("input,i", po::value(&sfmDataFilename)->required(), + "SfMData file.") + ("depthMapsFilterFolder", po::value(&depthMapsFilterFolder)->required(), + "Filtered depth maps folder.") + ("imagesFolder", po::value(&imagesFolder)->required(), + "Images used for depth map computation.\n" + "Filename should be the image uid.") + ("output,o", po::value(&outputFolder)->required(), + "Folder for output lighting vector files."); + + //po::options_description optionalParams("Optional parameters"); + //optionalParams.add_options(); + + po::options_description logParams("Log parameters"); + logParams.add_options() + ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), + "verbosity level (fatal, error, warning, info, debug, trace)."); + + allParams.add(requiredParams).add(logParams); //.add(optionalParams) + + po::variables_map vm; + try + { + po::store(po::parse_command_line(argc, argv, allParams), vm); + + if(vm.count("help") || (argc == 1)) + { + ALICEVISION_COUT(allParams); + return EXIT_SUCCESS; + } + po::notify(vm); + } + catch(boost::program_options::required_option& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + catch(boost::program_options::error& e) + { + ALICEVISION_CERR("ERROR: " << e.what()); + ALICEVISION_COUT("Usage:\n\n" << allParams); + return EXIT_FAILURE; + } + + ALICEVISION_COUT("Program called with the following parameters:"); + ALICEVISION_COUT(vm); + + // set verbose level + aliceVision::system::Logger::get()->setLogLevel(verboseLevel); + + // read the input SfM scene + sfmData::SfMData sfmData; + if(!sfmDataIO::Load(sfmData, sfmDataFilename, sfmDataIO::ESfMData::ALL)) + { + ALICEVISION_LOG_ERROR("The input SfMData file '" << sfmDataFilename << "' cannot be read."); + return EXIT_FAILURE; + } + + // initialization + mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, false); + + for(const auto& viewPair : sfmData.getViews()) + { + const IndexT viewId = viewPair.first; + + const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); + const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); + + lightingEstimation::LightingVector shl; + image::Image picture; + image::Image normals; + + image::readImage(picturePath, picture); + image::readImage(normalsPath, normals); + + lightingEstimation::estimateLigthing(shl, picture, picture, normals); + + std::ofstream file((fs::path(outputFolder) / (std::to_string(viewId) + ".shl")).string()); + if(file.is_open()) + file << shl; + } + + ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); + return EXIT_SUCCESS; +} From 9e56581bceff0b4d4a1c8ec3bf57f7f70c78825b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20De=20Lillo?= Date: Tue, 5 Feb 2019 11:01:13 +0100 Subject: [PATCH 2/6] [lightingEstimation] new options global/per_image and albedo input modes --- .../lightingEstimation/lightingEstimation.cpp | 147 +++++++-- .../lightingEstimation/lightingEstimation.hpp | 66 +++- .../lightingEstimation_test.cpp | 4 +- .../utils/main_lightingEstimation.cpp | 311 ++++++++++++++++-- 4 files changed, 465 insertions(+), 63 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingEstimation.cpp b/src/aliceVision/lightingEstimation/lightingEstimation.cpp index 66da573237..93c442ee6a 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation.cpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation.cpp @@ -7,6 +7,8 @@ #include "lightingEstimation.hpp" #include "augmentedNormals.hpp" +#include + #include #include @@ -20,67 +22,144 @@ namespace lightingEstimation { */ void albedoNormalsProduct(MatrixXf& rhoTimesN, const MatrixXf& albedoChannel, const image::Image& augmentedNormals) { + std::size_t validIndex = 0; for (int i = 0; i < augmentedNormals.size(); ++i) { - rhoTimesN(i, 0) = albedoChannel(i) * augmentedNormals(i).nx(); - rhoTimesN(i, 1) = albedoChannel(i) * augmentedNormals(i).ny(); - rhoTimesN(i, 2) = albedoChannel(i) * augmentedNormals(i).nz(); - rhoTimesN(i, 3) = albedoChannel(i) * augmentedNormals(i).nambiant(); - rhoTimesN(i, 4) = albedoChannel(i) * augmentedNormals(i).nx_ny(); - rhoTimesN(i, 5) = albedoChannel(i) * augmentedNormals(i).nx_nz(); - rhoTimesN(i, 6) = albedoChannel(i) * augmentedNormals(i).ny_nz(); - rhoTimesN(i, 7) = albedoChannel(i) * augmentedNormals(i).nx2_ny2(); - rhoTimesN(i, 8) = albedoChannel(i) * augmentedNormals(i).nz2(); + // If the normal is undefined + if(augmentedNormals(i).nx() == -1.0f && augmentedNormals(i).ny() == -1.0f && augmentedNormals(i).nz() == -1.0f) + { + continue; + } + rhoTimesN(validIndex, 0) = albedoChannel(i) * augmentedNormals(i).nx(); + rhoTimesN(validIndex, 1) = albedoChannel(i) * augmentedNormals(i).ny(); + rhoTimesN(validIndex, 2) = albedoChannel(i) * augmentedNormals(i).nz(); + rhoTimesN(validIndex, 3) = albedoChannel(i) * augmentedNormals(i).nambiant(); + rhoTimesN(validIndex, 4) = albedoChannel(i) * augmentedNormals(i).nx_ny(); + rhoTimesN(validIndex, 5) = albedoChannel(i) * augmentedNormals(i).nx_nz(); + rhoTimesN(validIndex, 6) = albedoChannel(i) * augmentedNormals(i).ny_nz(); + rhoTimesN(validIndex, 7) = albedoChannel(i) * augmentedNormals(i).nx2_ny2(); + rhoTimesN(validIndex, 8) = albedoChannel(i) * augmentedNormals(i).nz2(); + ++validIndex; } } /** * @brief Resolve lighting estimation problem for one channel */ - -void estimateLigthingOneChannel(Eigen::Matrix& lighting, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) +void estimateLigthing(Eigen::Matrix& lighting, const MatrixXf& rhoTimesN, const MatrixXf& pictureChannel) +{ + ALICEVISION_LOG_INFO("estimateLigthing: " << rhoTimesN.rows() << "x" << rhoTimesN.cols()); + lighting = rhoTimesN.colPivHouseholderQr().solve(pictureChannel); +} + +void prepareImageData(MatrixXf& rhoTimesN, MatrixXf& outColor, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) { - const auto nbPoints = augNormals.size(); + std::size_t nbValidPoints = 0; + for (int i = 0; i < augNormals.size(); ++i) + { + // If the normal is undefined + if(augNormals(i).nx() == -1.0f && augNormals(i).ny() == -1.0f && augNormals(i).nz() == -1.0f) + continue; + ++nbValidPoints; + } + + rhoTimesN.resize(nbValidPoints, 9); + albedoNormalsProduct(rhoTimesN, albedoChannel, augNormals); + + outColor.resize(nbValidPoints, 1); + { + std::size_t validIndex = 0; + for (int i = 0; i < augNormals.size(); ++i) + { + // If the normal is undefined + if(augNormals(i).nx() == -1.0f && augNormals(i).ny() == -1.0f && augNormals(i).nz() == -1.0f) + continue; - MatrixXf rhoTimesN(nbPoints, 9); - albedoNormalsProduct(rhoTimesN, albedoChannel, augNormals); + outColor(validIndex++) = pictureChannel(i); + } + } +} - lighting = rhoTimesN.colPivHouseholderQr().solve(pictureChannel); +void accumulateImageData(MatrixXf& all_rhoTimesN, MatrixXf& all_colors, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) +{ + MatrixXf rhoTimesN; + MatrixXf colors; + prepareImageData(rhoTimesN, colors, albedoChannel, pictureChannel, augNormals); + + all_rhoTimesN.resize(all_rhoTimesN.rows() + rhoTimesN.rows(), rhoTimesN.cols()); + all_colors.resize(all_colors.rows() + colors.rows(), colors.cols()); + + all_rhoTimesN.bottomLeftCorner(rhoTimesN.rows(), rhoTimesN.cols()) << rhoTimesN; + all_colors.bottomLeftCorner(colors.rows(), colors.cols()) << colors; +} + +void estimateLigthingOneChannel(Eigen::Matrix& lighting, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) +{ + MatrixXf rhoTimesN; + MatrixXf colors; + prepareImageData(rhoTimesN, colors, albedoChannel, pictureChannel, augNormals); + estimateLigthing(lighting, rhoTimesN, colors); } -void estimateLigthing(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals) +void estimateLigthingLuminance(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals) { using namespace Eigen; // Map albedo, image const std::size_t nbPixels = albedo.Width() * albedo.Height(); - Map> albedoR((float*)&albedo.data()->r(), nbPixels, 1); - Map> albedoG((float*)&albedo.data()->g(), nbPixels, 1); - Map> albedoB((float*)&albedo.data()->b(), nbPixels, 1); + // Augmented normales + image::Image augNormals(normals.cast()); - Map> pictureR((float*)&picture.data()->r(), nbPixels, 1); - Map> pictureG((float*)&picture.data()->g(), nbPixels, 1); - Map> pictureB((float*)&picture.data()->b(), nbPixels, 1); + MatrixXf rhoTimesN; + MatrixXf colors; - Eigen::Matrix lightingR; - Eigen::Matrix lightingG; - Eigen::Matrix lightingB; + // estimate Lighting per Channel + for(std::size_t c = 0; c < 3; ++c) + { + // Map albedo, image + Map> albedoC((float*)&(albedo(0,0)(c)), nbPixels, 1); + Map> pictureC((float*)&(picture(0,0)(c)), nbPixels, 1); - // Augmented normales - image::Image augNormals(normals.cast()); + accumulateImageData(rhoTimesN, colors, albedoC, pictureC, augNormals); - // EstimateLightingOneChannel - estimateLigthingOneChannel(lightingR, albedoR, pictureR, augNormals); - estimateLigthingOneChannel(lightingG, albedoG, pictureG, augNormals); - estimateLigthingOneChannel(lightingB, albedoB, pictureB, augNormals); + ALICEVISION_LOG_INFO("estimateLigthingLuminance (channel=" << c << "): " << rhoTimesN.rows() << "x" << rhoTimesN.cols()); + } + + Eigen::Matrix lightingL; + estimateLigthing(lightingL, rhoTimesN, colors); // lighting vectors fusion - lighting.col(0) = lightingR; - lighting.col(1) = lightingG; - lighting.col(2) = lightingB; + lighting.col(0) = lightingL; + lighting.col(1) = lightingL; + lighting.col(2) = lightingL; } +void estimateLigthingRGB(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals) +{ + using namespace Eigen; + + // Augmented normales + image::Image augNormals(normals.cast()); + + const std::size_t nbPixels = albedo.Width() * albedo.Height(); + + // estimate Lighting per Channel + for(std::size_t c = 0; c < 3; ++c) + { + // Map albedo, image + Map> albedoC((float*)&(albedo(0,0)(c)), nbPixels, 1); + Map> pictureC((float*)&(picture(0,0)(c)), nbPixels, 1); + + Eigen::Matrix lightingC; + + estimateLigthingOneChannel(lightingC, albedoC, pictureC, augNormals); + + // lighting vectors fusion + lighting.col(c) = lightingC; + } +} + } } diff --git a/src/aliceVision/lightingEstimation/lightingEstimation.hpp b/src/aliceVision/lightingEstimation/lightingEstimation.hpp index 253943fa54..839831b0fe 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation.hpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation.hpp @@ -21,16 +21,80 @@ namespace lightingEstimation { using Eigen::MatrixXf; +enum class ELightingColor { + Luminance = 1, + RGB = 3 +}; + +inline std::string ELightingColor_enumToString(ELightingColor v) +{ + switch(v) + { + case ELightingColor::RGB: + return "rgb"; + case ELightingColor::Luminance: + return "luminance"; + } + throw std::out_of_range("Invalid LightingColor type Enum: " + std::to_string(int(v))); +} + +inline ELightingColor ELightingColor_stringToEnum(const std::string& v) +{ + std::string vv = v; + std::transform(vv.begin(), vv.end(), vv.begin(), ::tolower); //tolower + + if(vv == "rgb") + return ELightingColor::RGB; + if(vv == "luminance") + return ELightingColor::Luminance; + throw std::out_of_range("Invalid LightingColor type string " + v); +} + +inline std::ostream& operator<<(std::ostream& os, ELightingColor v) +{ + return os << ELightingColor_enumToString(v); +} + +inline std::istream& operator>>(std::istream& in, ELightingColor& v) +{ + std::string token; + in >> token; + v = ELightingColor_stringToEnum(token); + return in; +} + /** * @brief Augmented lighting vetor for augmented Lambert's law (using Spherical Harmonics model) * Composed of 9 coefficients */ using LightingVector = Eigen::Matrix; +void estimateLigthing(Eigen::Matrix& lighting, const MatrixXf& rhoTimesN, const MatrixXf& pictureChannel); + /** * @brief Lighting estimation from picture, albedo and geometry */ -void estimateLigthing(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals); +void estimateLigthingLuminance(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals); + +/** + * @brief Lighting estimation from picture, albedo and geometry + */ +void estimateLigthingRGB(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals); + +inline void estimateLigthing(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals, ELightingColor lightingColor) +{ + switch(lightingColor) + { + case ELightingColor::RGB: + estimateLigthingRGB(lighting, albedo, picture, normals); + break; + case ELightingColor::Luminance: + estimateLigthingLuminance(lighting, albedo, picture, normals); + break; + } +} + +void accumulateImageData(MatrixXf& all_rhoTimesN, MatrixXf& all_colors, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals); } diff --git a/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp b/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp index fe8e3a1a6e..e7805db23b 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(LIGHTING_ESTIMATION_Lambertian_GT) // Retrieve unknown lighting LightingVector lightingEst; - estimateLigthing(lightingEst, albedoSynt, pictureGenerated, normalsSynt); + estimateLigthingRGB(lightingEst, albedoSynt, pictureGenerated, normalsSynt); const float epsilon = 1e-3f; EXPECT_MATRIX_NEAR(lightingEst, lightingSynt, epsilon); @@ -118,7 +118,7 @@ BOOST_AUTO_TEST_CASE(LIGHTING_ESTIMATION_Lambertian_noise) // Retrieve unknown lighting LightingVector lightingEst; - estimateLigthing(lightingEst, albedoSynt, pictureGenerated, normalsSynt); + estimateLigthingRGB(lightingEst, albedoSynt, pictureGenerated, normalsSynt); const float epsilon = 1e-2f; EXPECT_MATRIX_NEAR(lightingEst, lightingSynt, epsilon); diff --git a/src/software/utils/main_lightingEstimation.cpp b/src/software/utils/main_lightingEstimation.cpp index f1ce479a07..1ed30b1656 100644 --- a/src/software/utils/main_lightingEstimation.cpp +++ b/src/software/utils/main_lightingEstimation.cpp @@ -14,6 +14,9 @@ #include #include +#include +#include + #include #include @@ -30,6 +33,156 @@ using namespace aliceVision; namespace po = boost::program_options; namespace fs = boost::filesystem; +enum class EAlbedoEstimation +{ + CONSTANT, + PICTURE, + MEDIAN_FILTER, + BLUR_FILTER +}; + +/** + * @brief get informations about each describer type + * @return String + */ +std::string EAlbedoEstimation_informations() +{ + return "Albedo estimation method used for light estimation:\n" + "* constant: constant color.\n" + "* picture: from original picture.\n" + "* median_filter: from original picture with median filter.\n" + "* blur_filter: from original picture with blur filter.\n"; +} + +/** + * @brief convert an enum EAlbedoEstimation to its corresponding string + * @param EAlbedoEstimation + * @return String + */ +inline std::string EAlbedoEstimation_enumToString(EAlbedoEstimation albedoEstimation) +{ + switch(albedoEstimation) + { + case EAlbedoEstimation::CONSTANT: return "constant"; + case EAlbedoEstimation::PICTURE: return "picture"; + case EAlbedoEstimation::MEDIAN_FILTER: return "median_filter"; + case EAlbedoEstimation::BLUR_FILTER: return "blur_filter"; + } + throw std::out_of_range("Invalid albedoEstimation enum: " + std::to_string(int(albedoEstimation))); +} + +/** + * @brief convert a string albedoEstimation to its corresponding enum EAlbedoEstimation + * @param String + * @return EAlbedoEstimation + */ +inline EAlbedoEstimation EAlbedoEstimation_stringToEnum(const std::string& albedoEstimation) +{ + std::string albedo = albedoEstimation; + std::transform(albedo.begin(), albedo.end(), albedo.begin(), ::tolower); //tolower + + if(albedo == "constant") return EAlbedoEstimation::CONSTANT; + if(albedo == "picture") return EAlbedoEstimation::PICTURE; + if(albedo == "median_filter") return EAlbedoEstimation::MEDIAN_FILTER; + if(albedo == "blur_filter") return EAlbedoEstimation::BLUR_FILTER; + + throw std::out_of_range("Invalid albedoEstimation: " + albedoEstimation); +} + +inline std::ostream& operator<<(std::ostream& os, EAlbedoEstimation v) +{ + return os << EAlbedoEstimation_enumToString(v); +} + +inline std::istream& operator>>(std::istream& in, EAlbedoEstimation& v) +{ + std::string token; + in >> token; + v = EAlbedoEstimation_stringToEnum(token); + return in; +} + + + +enum class ELightingEstimationMode +{ + GLOBAL = 1, + PER_IMAGE = 2 +}; + +inline std::string ELightingEstimationMode_enumToString(ELightingEstimationMode v) +{ + switch(v) + { + case ELightingEstimationMode::GLOBAL: return "global"; + case ELightingEstimationMode::PER_IMAGE: return "per_image"; + } + throw std::out_of_range("Invalid LightEstimationMode enum: " + std::to_string(int(v))); +} + +inline ELightingEstimationMode ELightingEstimationMode_stringToEnum(const std::string& v) +{ + std::string vv = v; + std::transform(vv.begin(), vv.end(), vv.begin(), ::tolower); //tolower + + if(vv == "global") return ELightingEstimationMode::GLOBAL; + if(vv == "per_image") return ELightingEstimationMode::PER_IMAGE; + + throw std::out_of_range("Invalid LightEstimationMode: " + v); +} + +inline std::ostream& operator<<(std::ostream& os, ELightingEstimationMode v) +{ + return os << ELightingEstimationMode_enumToString(v); +} + +inline std::istream& operator>>(std::istream& in, ELightingEstimationMode& v) +{ + std::string token; + in >> token; + v = ELightingEstimationMode_stringToEnum(token); + return in; +} + + +void initAlbedo(image::Image& albedo, const image::Image& picture, EAlbedoEstimation albedoEstimationMethod, int albedoEstimationFilterSize, const std::string& outputFolder, IndexT viewId) +{ + switch(albedoEstimationMethod) + { + case EAlbedoEstimation::CONSTANT: + { + albedo.resize(picture.Width(), picture.Height()); + albedo.fill(image::RGBfColor(.5f,.5f,.5f)); + } + break; + case EAlbedoEstimation::PICTURE: + { + albedo = picture; + } + break; + case EAlbedoEstimation::MEDIAN_FILTER: + { + albedo.resize(picture.Width(), picture.Height()); + const oiio::ImageBuf pictureBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 3, oiio::TypeDesc::FLOAT), const_cast((void*)&picture(0,0)(0))); + oiio::ImageBuf albedoBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 3, oiio::TypeDesc::FLOAT), albedo.data()); + oiio::ImageBufAlgo::median_filter(albedoBuf, pictureBuf, albedoEstimationFilterSize, albedoEstimationFilterSize); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + } + break; + case EAlbedoEstimation::BLUR_FILTER: + { + albedo.resize(picture.Width(), picture.Height()); + const oiio::ImageBuf pictureBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 3, oiio::TypeDesc::FLOAT), const_cast((void*)&picture(0,0)(0))); + oiio::ImageBuf albedoBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 3, oiio::TypeDesc::FLOAT), albedo.data()); + oiio::ImageBuf K; + oiio::ImageBufAlgo::make_kernel(K, "gaussian", albedoEstimationFilterSize, albedoEstimationFilterSize); + oiio::ImageBufAlgo::convolve(albedoBuf, pictureBuf, K); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + } + break; + } +} + int main(int argc, char** argv) { system::Timer timer; @@ -42,29 +195,43 @@ int main(int argc, char** argv) std::string imagesFolder; std::string outputFolder; + EAlbedoEstimation albedoEstimationMethod = EAlbedoEstimation::CONSTANT; + ELightingEstimationMode lightEstimationMode = ELightingEstimationMode::PER_IMAGE; + + int albedoEstimationFilterSize = 3; + lightingEstimation::ELightingColor lightingColor = lightingEstimation::ELightingColor::RGB; + po::options_description allParams("AliceVision lighthingEstimation"); po::options_description requiredParams("Required parameters"); requiredParams.add_options() - ("input,i", po::value(&sfmDataFilename)->required(), - "SfMData file.") - ("depthMapsFilterFolder", po::value(&depthMapsFilterFolder)->required(), - "Filtered depth maps folder.") - ("imagesFolder", po::value(&imagesFolder)->required(), - "Images used for depth map computation.\n" - "Filename should be the image uid.") - ("output,o", po::value(&outputFolder)->required(), - "Folder for output lighting vector files."); - - //po::options_description optionalParams("Optional parameters"); - //optionalParams.add_options(); - + ("input,i", po::value(&sfmDataFilename)->required(), + "SfMData file.") + ("depthMapsFilterFolder", po::value(&depthMapsFilterFolder)->required(), + "Filtered depth maps folder.") + ("imagesFolder", po::value(&imagesFolder)->required(), + "Images used for depth map computation.\n" + "Filename should be the image uid.") + ("output,o", po::value(&outputFolder)->required(), + "Folder for output lighting vector files."); + + po::options_description optionalParams("Optional parameters"); + optionalParams.add_options() + ("lightingColor", po::value(&lightingColor)->default_value(lightingColor), + "Lighting color.") + ("lightingEstimationMode", po::value(&lightEstimationMode)->default_value(lightEstimationMode), + "Lighting Estimation Mode.") + ("albedoEstimationName", po::value(&albedoEstimationMethod)->default_value(albedoEstimationMethod), + EAlbedoEstimation_informations().c_str()) + ("albedoEstimationFilterSize", po::value(&albedoEstimationFilterSize)->default_value(albedoEstimationFilterSize), + "Albedo filter size for estimation method using filter."); + po::options_description logParams("Log parameters"); logParams.add_options() ("verboseLevel,v", po::value(&verboseLevel)->default_value(verboseLevel), "verbosity level (fatal, error, warning, info, debug, trace)."); - allParams.add(requiredParams).add(logParams); //.add(optionalParams) + allParams.add(requiredParams).add(optionalParams).add(logParams); po::variables_map vm; try @@ -108,27 +275,119 @@ int main(int argc, char** argv) // initialization mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, false); - for(const auto& viewPair : sfmData.getViews()) + if(lightEstimationMode == ELightingEstimationMode::PER_IMAGE) + { + for(const auto& viewPair : sfmData.getViews()) + { + const IndexT viewId = viewPair.first; + + const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); + const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); + + lightingEstimation::LightingVector shl; + image::Image picture; + image::Image normals; + + image::readImage(picturePath, picture); + image::readImage(normalsPath, normals); + + image::Image albedo; + initAlbedo(albedo, picture, albedoEstimationMethod, albedoEstimationFilterSize, outputFolder, viewId); + + lightingEstimation::estimateLigthing(shl, albedo, picture, normals, lightingColor); + + std::ofstream file((fs::path(outputFolder) / (std::to_string(viewId) + ".shl")).string()); + if(file.is_open()) + file << shl; + } + } + else if(lightEstimationMode == ELightingEstimationMode::GLOBAL) { - const IndexT viewId = viewPair.first; + using namespace Eigen; + using namespace aliceVision::lightingEstimation; + + MatrixXf rhoTimesN[3]; + MatrixXf colors[3]; + + for(const auto& viewPair : sfmData.getViews()) + { + const IndexT viewId = viewPair.first; + + const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); + const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); + + image::Image picture; + image::Image normals; - const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); - const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); + image::readImage(picturePath, picture); + image::readImage(normalsPath, normals); - lightingEstimation::LightingVector shl; - image::Image picture; - image::Image normals; + image::Image albedo; + initAlbedo(albedo, picture, albedoEstimationMethod, albedoEstimationFilterSize, outputFolder, viewId); - image::readImage(picturePath, picture); - image::readImage(normalsPath, normals); + // Augmented normales + image::Image augNormals(normals.cast()); - lightingEstimation::estimateLigthing(shl, picture, picture, normals); + const std::size_t nbPixels = albedo.Width() * albedo.Height(); + + // estimate Lighting per Channel + for(std::size_t c = 0; c < 3; ++c) + { + // Map albedo, image + Map> albedoC((float*)&(albedo(0,0)(c)), nbPixels, 1); + Map> pictureC((float*)&(picture(0,0)(c)), nbPixels, 1); + + accumulateImageData(rhoTimesN[c], colors[c], albedoC, pictureC, augNormals); + + ALICEVISION_LOG_INFO("GLOBAL: estimateLigthingLuminance (channel=" << c << "): " << rhoTimesN[c].rows() << "x" << rhoTimesN[c].cols()); + } + } + + LightingVector shl; + if(lightingColor == ELightingColor::Luminance) + { + MatrixXf all_rhoTimesN; + MatrixXf all_colors; - std::ofstream file((fs::path(outputFolder) / (std::to_string(viewId) + ".shl")).string()); + for(std::size_t c = 0; c < 3; ++c) + { + all_rhoTimesN.resize(all_rhoTimesN.rows() + rhoTimesN[c].rows(), rhoTimesN[c].cols()); + all_colors.resize(all_colors.rows() + colors[c].rows(), colors[c].cols()); + + all_rhoTimesN.bottomLeftCorner(rhoTimesN[c].rows(), rhoTimesN[c].cols()) = rhoTimesN[c]; + all_colors.bottomLeftCorner(colors[c].rows(), colors[c].cols()) = colors[c]; + } + + Eigen::Matrix lightingL; + estimateLigthing(lightingL, all_rhoTimesN, all_colors); + + // lighting vectors fusion + shl.col(0) = lightingL; + shl.col(1) = lightingL; + shl.col(2) = lightingL; + } + else if(lightingColor == ELightingColor::RGB) + { + // estimate Lighting per Channel + for(std::size_t c = 0; c < 3; ++c) + { + Eigen::Matrix lightingC; + estimateLigthing(lightingC, rhoTimesN[c], colors[c]); + + // lighting vectors fusion + shl.col(c) = lightingC; + } + } + + std::ofstream file((fs::path(outputFolder) / ("scene.shl")).string()); if(file.is_open()) file << shl; } - + else + { + throw std::runtime_error("Invalid lightingEstimationMode: " + ELightingEstimationMode_enumToString(lightEstimationMode)); + } + ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); return EXIT_SUCCESS; } From d3adbf885f0f527c1e64475d7e2ef851715ca28f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20De=20Lillo?= Date: Tue, 12 Feb 2019 14:04:51 +0100 Subject: [PATCH 3/6] [image] `pixelTypes` Add method `Channels` --- src/aliceVision/image/Image.hpp | 14 ++++++++++++-- src/aliceVision/image/pixelTypes.hpp | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/image/Image.hpp b/src/aliceVision/image/Image.hpp index 7d69d0db63..beeb406809 100644 --- a/src/aliceVision/image/Image.hpp +++ b/src/aliceVision/image/Image.hpp @@ -7,7 +7,8 @@ #pragma once -#include "aliceVision/numeric/numeric.hpp" +#include +#include //--------------------------------- // Universal Image Processing Algorithm @@ -23,7 +24,6 @@ namespace aliceVision { namespace image { - template class Image : public Eigen::Matrix { @@ -143,6 +143,16 @@ namespace aliceVision return sizeof( Tpixel ); } + + /** + * @brief Return the number of channels + * @return number of channels + */ + inline int Channels() const + { + return NbChannels::size; + } + /** * @brief constant random pixel access * @param y Index of the row diff --git a/src/aliceVision/image/pixelTypes.hpp b/src/aliceVision/image/pixelTypes.hpp index ad8c9036e7..a4773d0efd 100644 --- a/src/aliceVision/image/pixelTypes.hpp +++ b/src/aliceVision/image/pixelTypes.hpp @@ -394,6 +394,8 @@ namespace aliceVision } }; + typedef Rgba RGBAColor; + /// Instantiation for unsigned char color component using RGBAColor = Rgba; /// Instantiation for float color component @@ -410,6 +412,20 @@ namespace aliceVision const RGBfColor FWHITE(1.0f, 1.0f, 1.0f); const RGBfColor FBLACK( .0f, .0f, .0f); + + template + struct NbChannels + { + // no size parameter, so no default value. + // An error will be raise at compile time if this type traits is not defined. + }; + + template<> struct NbChannels{ int size = 1; }; + template<> struct NbChannels{ int size = 1; }; + template<> struct NbChannels{ int size = 3; }; + template<> struct NbChannels{ int size = 3; }; + template<> struct NbChannels{ int size = 4; }; + } // namespace image } // namespace aliceVision From 3b8ae49a699e6ae16c6046bbf06059f3482e5a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20De=20Lillo?= Date: Tue, 12 Feb 2019 14:06:28 +0100 Subject: [PATCH 4/6] [lightingEstimator] Use a new class `LightingEstimator` instead of multiple functions --- .../lightingEstimation/lightingEstimation.cpp | 181 +++++++++------ .../lightingEstimation/lightingEstimation.hpp | 105 +++------ .../lightingEstimation_test.cpp | 10 +- .../utils/main_lightingEstimation.cpp | 209 ++++++++++-------- 4 files changed, 270 insertions(+), 235 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingEstimation.cpp b/src/aliceVision/lightingEstimation/lightingEstimation.cpp index 93c442ee6a..7e22c1f2f7 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation.cpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation.cpp @@ -43,123 +43,164 @@ void albedoNormalsProduct(MatrixXf& rhoTimesN, const MatrixXf& albedoChannel, co } } -/** - * @brief Resolve lighting estimation problem for one channel - */ -void estimateLigthing(Eigen::Matrix& lighting, const MatrixXf& rhoTimesN, const MatrixXf& pictureChannel) -{ - ALICEVISION_LOG_INFO("estimateLigthing: " << rhoTimesN.rows() << "x" << rhoTimesN.cols()); - lighting = rhoTimesN.colPivHouseholderQr().solve(pictureChannel); -} - -void prepareImageData(MatrixXf& rhoTimesN, MatrixXf& outColor, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) +void prepareImageData(const MatrixXf& albedo, + const MatrixXf& picture, + const image::Image& augmentedNormals, + MatrixXf& rhoTimesN, + MatrixXf& colors) { std::size_t nbValidPoints = 0; - for (int i = 0; i < augNormals.size(); ++i) + + for(int i = 0; i < augmentedNormals.size(); ++i) { - // If the normal is undefined - if(augNormals(i).nx() == -1.0f && augNormals(i).ny() == -1.0f && augNormals(i).nz() == -1.0f) + // if the normal is undefined + if(augmentedNormals(i).nx() == -1.0f && augmentedNormals(i).ny() == -1.0f && augmentedNormals(i).nz() == -1.0f) continue; ++nbValidPoints; } rhoTimesN.resize(nbValidPoints, 9); - albedoNormalsProduct(rhoTimesN, albedoChannel, augNormals); + albedoNormalsProduct(rhoTimesN, albedo, augmentedNormals); + colors.resize(nbValidPoints, 1); - outColor.resize(nbValidPoints, 1); { std::size_t validIndex = 0; - for (int i = 0; i < augNormals.size(); ++i) + for(int i = 0; i < augmentedNormals.size(); ++i) { - // If the normal is undefined - if(augNormals(i).nx() == -1.0f && augNormals(i).ny() == -1.0f && augNormals(i).nz() == -1.0f) + // if the normal is undefined + if(augmentedNormals(i).nx() == -1.0f && augmentedNormals(i).ny() == -1.0f && augmentedNormals(i).nz() == -1.0f) continue; - outColor(validIndex++) = pictureChannel(i); + colors(validIndex++) = picture(i); } } } -void accumulateImageData(MatrixXf& all_rhoTimesN, MatrixXf& all_colors, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) +void LighthingEstimator::addImage(const image::Image& albedo, const image::Image& picture, const image::Image& normals) { - MatrixXf rhoTimesN; - MatrixXf colors; - prepareImageData(rhoTimesN, colors, albedoChannel, pictureChannel, augNormals); + using namespace Eigen; - all_rhoTimesN.resize(all_rhoTimesN.rows() + rhoTimesN.rows(), rhoTimesN.cols()); - all_colors.resize(all_colors.rows() + colors.rows(), colors.cols()); + // augmented normales + image::Image augmentedNormals(normals.cast()); - all_rhoTimesN.bottomLeftCorner(rhoTimesN.rows(), rhoTimesN.cols()) << rhoTimesN; - all_colors.bottomLeftCorner(colors.rows(), colors.cols()) << colors; -} + const std::size_t nbPixels = augmentedNormals.Width() * augmentedNormals.Height(); -void estimateLigthingOneChannel(Eigen::Matrix& lighting, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals) -{ MatrixXf rhoTimesN; MatrixXf colors; - prepareImageData(rhoTimesN, colors, albedoChannel, pictureChannel, augNormals); - estimateLigthing(lighting, rhoTimesN, colors); + + // map albedo, image + Map channelAlbedo(albedo.data(), nbPixels, 1); + Map channelPicture(picture.data(), nbPixels, 1); + + prepareImageData(channelAlbedo, channelPicture, augmentedNormals, rhoTimesN, colors); + + // store image data + _all_rhoTimesN.at(0).push_back(rhoTimesN); + _all_pictureChannel.at(0).push_back(colors); } -void estimateLigthingLuminance(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals) +void LighthingEstimator::addImage(const image::Image& albedo, const image::Image& picture, const image::Image& normals) { - using namespace Eigen; + using namespace Eigen; - // Map albedo, image - const std::size_t nbPixels = albedo.Width() * albedo.Height(); + // augmented normales + image::Image augmentedNormals(normals.cast()); - // Augmented normales - image::Image augNormals(normals.cast()); + const std::size_t nbPixels = augmentedNormals.Width() * augmentedNormals.Height(); + // estimate lighting per channel + for(std::size_t channel = 0; channel < 3; ++channel) + { MatrixXf rhoTimesN; MatrixXf colors; - // estimate Lighting per Channel - for(std::size_t c = 0; c < 3; ++c) - { - // Map albedo, image - Map> albedoC((float*)&(albedo(0,0)(c)), nbPixels, 1); - Map> pictureC((float*)&(picture(0,0)(c)), nbPixels, 1); + // map albedo, image + Map> channelAlbedo(static_cast(&(albedo(0,0)(channel))), nbPixels, 1); + Map> channelPicture(static_cast(&(picture(0,0)(channel))), nbPixels, 1); - accumulateImageData(rhoTimesN, colors, albedoC, pictureC, augNormals); + prepareImageData(channelAlbedo, channelPicture, augmentedNormals, rhoTimesN, colors); - ALICEVISION_LOG_INFO("estimateLigthingLuminance (channel=" << c << "): " << rhoTimesN.rows() << "x" << rhoTimesN.cols()); - } + // store image data + _all_rhoTimesN.at(channel).push_back(rhoTimesN); + _all_pictureChannel.at(channel).push_back(colors); + } +} - Eigen::Matrix lightingL; - estimateLigthing(lightingL, rhoTimesN, colors); +void LighthingEstimator::estimateLigthing(LightingVector& lighting) const +{ + int nbChannels = 3; - // lighting vectors fusion - lighting.col(0) = lightingL; - lighting.col(1) = lightingL; - lighting.col(2) = lightingL; -} + // check number of channels + for(int channel = 0; channel < 3; ++channel) + { + if(_all_rhoTimesN.at(channel).empty()) + { + nbChannels = channel; + break; + } + } + // for each channel + for(int channel = 0; channel < nbChannels; ++channel) + { + std::size_t nbRows_all_rhoTimesN = 0; + std::size_t nbRows_all_pictureChannel = 0; -void estimateLigthingRGB(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals) -{ - using namespace Eigen; + // count and check matrices rows + { + for(const MatrixXf& matrix : _all_rhoTimesN.at(channel)) + nbRows_all_rhoTimesN += matrix.rows(); + + for(const MatrixXf& matrix : _all_pictureChannel.at(channel)) + nbRows_all_pictureChannel += matrix.rows(); - // Augmented normales - image::Image augNormals(normals.cast()); + assert(nbRows_all_rhoTimesN == nbRows_all_pictureChannel); + } - const std::size_t nbPixels = albedo.Width() * albedo.Height(); + // agglomerate rhoTimesN + MatrixXf rhoTimesN(nbRows_all_rhoTimesN, 9); + { + std::size_t nbRow = 0; + for(const MatrixXf& matrix : _all_rhoTimesN.at(channel)) + { + for(int matrixRow = 0; matrixRow < matrix.rows(); ++matrixRow) + rhoTimesN.row(nbRow + matrixRow) = matrix.row(matrixRow); + nbRow += matrix.rows(); + } + } - // estimate Lighting per Channel - for(std::size_t c = 0; c < 3; ++c) + // agglomerate pictureChannel + MatrixXf pictureChannel(nbRows_all_pictureChannel, 1); { - // Map albedo, image - Map> albedoC((float*)&(albedo(0,0)(c)), nbPixels, 1); - Map> pictureC((float*)&(picture(0,0)(c)), nbPixels, 1); + std::size_t nbRow = 0; + for(const MatrixXf& matrix : _all_pictureChannel.at(channel)) + { + for(int matrixRow = 0; matrixRow < matrix.rows(); ++matrixRow) + pictureChannel.row(nbRow + matrixRow) = matrix.row(matrixRow); + nbRow += matrix.rows(); + } + } - Eigen::Matrix lightingC; + ALICEVISION_LOG_INFO("estimate ligthing channel: rhoTimesN(" << rhoTimesN.rows() << "x" << rhoTimesN.cols()<< ")"); + Eigen::Matrix lightingC = rhoTimesN.colPivHouseholderQr().solve(pictureChannel); - estimateLigthingOneChannel(lightingC, albedoC, pictureC, augNormals); + // lighting vectors fusion + lighting.col(channel) = lightingC; - // lighting vectors fusion - lighting.col(c) = lightingC; + // luminance estimation + if(nbChannels == 1) + { + lighting.col(1) = lightingC; + lighting.col(2) = lightingC; } + } } +void LighthingEstimator::clear() +{ + _all_rhoTimesN = std::array, 3>(); + _all_pictureChannel = std::array, 3>(); } -} + +} // namespace lightingEstimation +} // namespace aliceVision diff --git a/src/aliceVision/lightingEstimation/lightingEstimation.hpp b/src/aliceVision/lightingEstimation/lightingEstimation.hpp index 839831b0fe..c4a6ad9d0d 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation.hpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation.hpp @@ -6,10 +6,9 @@ #pragma once -#include "augmentedNormals.hpp" - #include #include +#include #include #include @@ -21,81 +20,47 @@ namespace lightingEstimation { using Eigen::MatrixXf; -enum class ELightingColor { - Luminance = 1, - RGB = 3 -}; - -inline std::string ELightingColor_enumToString(ELightingColor v) -{ - switch(v) - { - case ELightingColor::RGB: - return "rgb"; - case ELightingColor::Luminance: - return "luminance"; - } - throw std::out_of_range("Invalid LightingColor type Enum: " + std::to_string(int(v))); -} - -inline ELightingColor ELightingColor_stringToEnum(const std::string& v) -{ - std::string vv = v; - std::transform(vv.begin(), vv.end(), vv.begin(), ::tolower); //tolower - - if(vv == "rgb") - return ELightingColor::RGB; - if(vv == "luminance") - return ELightingColor::Luminance; - throw std::out_of_range("Invalid LightingColor type string " + v); -} - -inline std::ostream& operator<<(std::ostream& os, ELightingColor v) -{ - return os << ELightingColor_enumToString(v); -} - -inline std::istream& operator>>(std::istream& in, ELightingColor& v) -{ - std::string token; - in >> token; - v = ELightingColor_stringToEnum(token); - return in; -} - /** * @brief Augmented lighting vetor for augmented Lambert's law (using Spherical Harmonics model) * Composed of 9 coefficients */ using LightingVector = Eigen::Matrix; -void estimateLigthing(Eigen::Matrix& lighting, const MatrixXf& rhoTimesN, const MatrixXf& pictureChannel); - -/** - * @brief Lighting estimation from picture, albedo and geometry - */ -void estimateLigthingLuminance(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals); - /** - * @brief Lighting estimation from picture, albedo and geometry + * @brief The LighthingEstimator class + * Allows to estimate LightingVector from a single image or multiple images + * @warning Image pixel type can be: + * - RGB (float) for light and color estimation + * - Greyscale (float) for luminance estimation */ -void estimateLigthingRGB(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals); - -inline void estimateLigthing(LightingVector& lighting, const image::Image& albedo, const image::Image& picture, const image::Image& normals, ELightingColor lightingColor) +class LighthingEstimator { - switch(lightingColor) - { - case ELightingColor::RGB: - estimateLigthingRGB(lighting, albedo, picture, normals); - break; - case ELightingColor::Luminance: - estimateLigthingLuminance(lighting, albedo, picture, normals); - break; - } -} - -void accumulateImageData(MatrixXf& all_rhoTimesN, MatrixXf& all_colors, const MatrixXf& albedoChannel, const MatrixXf& pictureChannel, const image::Image& augNormals); - +public: + + /** + * @brief Aggregate image data + * @param albedo[in] the corresponding albedo image + * @param picture[in] the corresponding picture + * @param normals[in] the corresponding normals image + */ + void addImage(const image::Image& albedo, const image::Image& picture, const image::Image& normals); + void addImage(const image::Image& albedo, const image::Image& picture, const image::Image& normals); + + /** + * @brief Estimate ligthing from the aggregate image(s) data + * @param[out] lighting Estimate ligthing @see LightingVector + */ + void estimateLigthing(LightingVector& lighting) const; + + /** + * @brief Clear all the aggregate image(s) data + */ + void clear(); + +private: + std::array, 3> _all_rhoTimesN; + std::array, 3> _all_pictureChannel; +}; -} -} +} // namespace lightingEstimation +} // namespace aliceVision diff --git a/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp b/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp index e7805db23b..9947628cea 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation_test.cpp @@ -67,8 +67,11 @@ BOOST_AUTO_TEST_CASE(LIGHTING_ESTIMATION_Lambertian_GT) } // Retrieve unknown lighting + LighthingEstimator estimator; LightingVector lightingEst; - estimateLigthingRGB(lightingEst, albedoSynt, pictureGenerated, normalsSynt); + + estimator.addImage(albedoSynt, pictureGenerated, normalsSynt); + estimator.estimateLigthing(lightingEst); const float epsilon = 1e-3f; EXPECT_MATRIX_NEAR(lightingEst, lightingSynt, epsilon); @@ -117,8 +120,11 @@ BOOST_AUTO_TEST_CASE(LIGHTING_ESTIMATION_Lambertian_noise) } // Retrieve unknown lighting + LighthingEstimator estimator; LightingVector lightingEst; - estimateLigthingRGB(lightingEst, albedoSynt, pictureGenerated, normalsSynt); + + estimator.addImage(albedoSynt, pictureGenerated, normalsSynt); + estimator.estimateLigthing(lightingEst); const float epsilon = 1e-2f; EXPECT_MATRIX_NEAR(lightingEst, lightingSynt, epsilon); diff --git a/src/software/utils/main_lightingEstimation.cpp b/src/software/utils/main_lightingEstimation.cpp index 1ed30b1656..112df6db27 100644 --- a/src/software/utils/main_lightingEstimation.cpp +++ b/src/software/utils/main_lightingEstimation.cpp @@ -102,8 +102,6 @@ inline std::istream& operator>>(std::istream& in, EAlbedoEstimation& v) return in; } - - enum class ELightingEstimationMode { GLOBAL = 1, @@ -115,7 +113,7 @@ inline std::string ELightingEstimationMode_enumToString(ELightingEstimationMode switch(v) { case ELightingEstimationMode::GLOBAL: return "global"; - case ELightingEstimationMode::PER_IMAGE: return "per_image"; + case ELightingEstimationMode::PER_IMAGE: return "per_image"; } throw std::out_of_range("Invalid LightEstimationMode enum: " + std::to_string(int(v))); } @@ -126,7 +124,7 @@ inline ELightingEstimationMode ELightingEstimationMode_stringToEnum(const std::s std::transform(vv.begin(), vv.end(), vv.begin(), ::tolower); //tolower if(vv == "global") return ELightingEstimationMode::GLOBAL; - if(vv == "per_image") return ELightingEstimationMode::PER_IMAGE; + if(vv == "per_image") return ELightingEstimationMode::PER_IMAGE; throw std::out_of_range("Invalid LightEstimationMode: " + v); } @@ -144,6 +142,47 @@ inline std::istream& operator>>(std::istream& in, ELightingEstimationMode& v) return in; } +enum class ELightingColor { + Luminance = 1, + RGB = 3 +}; + +inline std::string ELightingColor_enumToString(ELightingColor v) +{ + switch(v) + { + case ELightingColor::RGB: + return "rgb"; + case ELightingColor::Luminance: + return "luminance"; + } + throw std::out_of_range("Invalid LightingColor type Enum: " + std::to_string(int(v))); +} + +inline ELightingColor ELightingColor_stringToEnum(const std::string& v) +{ + std::string vv = v; + std::transform(vv.begin(), vv.end(), vv.begin(), ::tolower); //tolower + + if(vv == "rgb") + return ELightingColor::RGB; + if(vv == "luminance") + return ELightingColor::Luminance; + throw std::out_of_range("Invalid LightingColor type string " + v); +} + +inline std::ostream& operator<<(std::ostream& os, ELightingColor v) +{ + return os << ELightingColor_enumToString(v); +} + +inline std::istream& operator>>(std::istream& in, ELightingColor& v) +{ + std::string token; + in >> token; + v = ELightingColor_stringToEnum(token); + return in; +} void initAlbedo(image::Image& albedo, const image::Image& picture, EAlbedoEstimation albedoEstimationMethod, int albedoEstimationFilterSize, const std::string& outputFolder, IndexT viewId) { @@ -183,6 +222,44 @@ void initAlbedo(image::Image& albedo, const image::Image& albedo, const image::Image& picture, EAlbedoEstimation albedoEstimationMethod, int albedoEstimationFilterSize, const std::string& outputFolder, IndexT viewId) +{ + switch(albedoEstimationMethod) + { + case EAlbedoEstimation::CONSTANT: + { + albedo.resize(picture.Width(), picture.Height()); + albedo.fill(.5f); + } + break; + case EAlbedoEstimation::PICTURE: + { + albedo = picture; + } + break; + case EAlbedoEstimation::MEDIAN_FILTER: + { + albedo.resize(picture.Width(), picture.Height()); + const oiio::ImageBuf pictureBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 1, oiio::TypeDesc::FLOAT), const_cast(picture.data())); + oiio::ImageBuf albedoBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 1, oiio::TypeDesc::FLOAT), albedo.data()); + oiio::ImageBufAlgo::median_filter(albedoBuf, pictureBuf, albedoEstimationFilterSize, albedoEstimationFilterSize); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + } + break; + case EAlbedoEstimation::BLUR_FILTER: + { + albedo.resize(picture.Width(), picture.Height()); + const oiio::ImageBuf pictureBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 1, oiio::TypeDesc::FLOAT), const_cast(picture.data())); + oiio::ImageBuf albedoBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 1, oiio::TypeDesc::FLOAT), albedo.data()); + oiio::ImageBuf K; + oiio::ImageBufAlgo::make_kernel(K, "gaussian", albedoEstimationFilterSize, albedoEstimationFilterSize); + oiio::ImageBufAlgo::convolve(albedoBuf, pictureBuf, K); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + } + break; + } +} + int main(int argc, char** argv) { system::Timer timer; @@ -199,7 +276,7 @@ int main(int argc, char** argv) ELightingEstimationMode lightEstimationMode = ELightingEstimationMode::PER_IMAGE; int albedoEstimationFilterSize = 3; - lightingEstimation::ELightingColor lightingColor = lightingEstimation::ELightingColor::RGB; + ELightingColor lightingColor = ELightingColor::RGB; po::options_description allParams("AliceVision lighthingEstimation"); @@ -217,7 +294,7 @@ int main(int argc, char** argv) po::options_description optionalParams("Optional parameters"); optionalParams.add_options() - ("lightingColor", po::value(&lightingColor)->default_value(lightingColor), + ("lightingColor", po::value(&lightingColor)->default_value(lightingColor), "Lighting color.") ("lightingEstimationMode", po::value(&lightEstimationMode)->default_value(lightEstimationMode), "Lighting Estimation Mode.") @@ -275,118 +352,64 @@ int main(int argc, char** argv) // initialization mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, false); - if(lightEstimationMode == ELightingEstimationMode::PER_IMAGE) + lightingEstimation::LighthingEstimator estimator; + + for(const auto& viewPair : sfmData.getViews()) { - for(const auto& viewPair : sfmData.getViews()) - { - const IndexT viewId = viewPair.first; + const IndexT viewId = viewPair.first; - const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); - const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); + const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); + const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); - lightingEstimation::LightingVector shl; - image::Image picture; - image::Image normals; + image::Image normals; + image::readImage(normalsPath, normals); + if(lightingColor == ELightingColor::Luminance) + { + image::Image albedo, picture; image::readImage(picturePath, picture); - image::readImage(normalsPath, normals); - image::Image albedo; initAlbedo(albedo, picture, albedoEstimationMethod, albedoEstimationFilterSize, outputFolder, viewId); - lightingEstimation::estimateLigthing(shl, albedo, picture, normals, lightingColor); - - std::ofstream file((fs::path(outputFolder) / (std::to_string(viewId) + ".shl")).string()); - if(file.is_open()) - file << shl; + estimator.addImage(albedo, picture, normals); } - } - else if(lightEstimationMode == ELightingEstimationMode::GLOBAL) - { - using namespace Eigen; - using namespace aliceVision::lightingEstimation; - - MatrixXf rhoTimesN[3]; - MatrixXf colors[3]; - - for(const auto& viewPair : sfmData.getViews()) + else if(lightingColor == ELightingColor::RGB) { - const IndexT viewId = viewPair.first; - - const std::string picturePath = mp.getImagePath(mp.getIndexFromViewId(viewId)); - const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); - - image::Image picture; - image::Image normals; - + image::Image albedo, picture; image::readImage(picturePath, picture); - image::readImage(normalsPath, normals); - image::Image albedo; initAlbedo(albedo, picture, albedoEstimationMethod, albedoEstimationFilterSize, outputFolder, viewId); - // Augmented normales - image::Image augNormals(normals.cast()); - - const std::size_t nbPixels = albedo.Width() * albedo.Height(); - - // estimate Lighting per Channel - for(std::size_t c = 0; c < 3; ++c) - { - // Map albedo, image - Map> albedoC((float*)&(albedo(0,0)(c)), nbPixels, 1); - Map> pictureC((float*)&(picture(0,0)(c)), nbPixels, 1); - - accumulateImageData(rhoTimesN[c], colors[c], albedoC, pictureC, augNormals); - - ALICEVISION_LOG_INFO("GLOBAL: estimateLigthingLuminance (channel=" << c << "): " << rhoTimesN[c].rows() << "x" << rhoTimesN[c].cols()); - } + estimator.addImage(albedo, picture, normals); } - LightingVector shl; - if(lightingColor == ELightingColor::Luminance) + if(lightEstimationMode == ELightingEstimationMode::PER_IMAGE) { - MatrixXf all_rhoTimesN; - MatrixXf all_colors; - - for(std::size_t c = 0; c < 3; ++c) - { - all_rhoTimesN.resize(all_rhoTimesN.rows() + rhoTimesN[c].rows(), rhoTimesN[c].cols()); - all_colors.resize(all_colors.rows() + colors[c].rows(), colors[c].cols()); - - all_rhoTimesN.bottomLeftCorner(rhoTimesN[c].rows(), rhoTimesN[c].cols()) = rhoTimesN[c]; - all_colors.bottomLeftCorner(colors[c].rows(), colors[c].cols()) = colors[c]; - } - - Eigen::Matrix lightingL; - estimateLigthing(lightingL, all_rhoTimesN, all_colors); + ALICEVISION_LOG_INFO("Solving view: " << viewId << " lighting estimation..."); + lightingEstimation::LightingVector shl; + estimator.estimateLigthing(shl); + estimator.clear(); // clear aggregate data - // lighting vectors fusion - shl.col(0) = lightingL; - shl.col(1) = lightingL; - shl.col(2) = lightingL; + std::ofstream file((fs::path(outputFolder) / (std::to_string(viewId) + ".shl")).string()); + if(file.is_open()) + file << shl; } - else if(lightingColor == ELightingColor::RGB) + else { - // estimate Lighting per Channel - for(std::size_t c = 0; c < 3; ++c) - { - Eigen::Matrix lightingC; - estimateLigthing(lightingC, rhoTimesN[c], colors[c]); - - // lighting vectors fusion - shl.col(c) = lightingC; - } + ALICEVISION_LOG_INFO("View: " << viewId << " added to the problem."); } + } + + if(lightEstimationMode == ELightingEstimationMode::GLOBAL) + { + ALICEVISION_LOG_INFO("Solving global scene lighting estimation..."); + lightingEstimation::LightingVector shl; + estimator.estimateLigthing(shl); - std::ofstream file((fs::path(outputFolder) / ("scene.shl")).string()); + std::ofstream file((fs::path(outputFolder) / ("global.shl")).string()); if(file.is_open()) file << shl; } - else - { - throw std::runtime_error("Invalid lightingEstimationMode: " + ELightingEstimationMode_enumToString(lightEstimationMode)); - } ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); return EXIT_SUCCESS; From 577586766a83c8f518ea9d41dbd6ce9afab0e0a3 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 15 May 2020 23:40:58 +0200 Subject: [PATCH 5/6] [software] lightingEstimation: build fix regarding image io --- src/software/utils/main_lightingEstimation.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/software/utils/main_lightingEstimation.cpp b/src/software/utils/main_lightingEstimation.cpp index 112df6db27..eb0a30661d 100644 --- a/src/software/utils/main_lightingEstimation.cpp +++ b/src/software/utils/main_lightingEstimation.cpp @@ -205,7 +205,8 @@ void initAlbedo(image::Image& albedo, const image::Image((void*)&picture(0,0)(0))); oiio::ImageBuf albedoBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 3, oiio::TypeDesc::FLOAT), albedo.data()); oiio::ImageBufAlgo::median_filter(albedoBuf, pictureBuf, albedoEstimationFilterSize, albedoEstimationFilterSize); - image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo, + image::EImageColorSpace::AUTO); } break; case EAlbedoEstimation::BLUR_FILTER: @@ -216,7 +217,8 @@ void initAlbedo(image::Image& albedo, const image::Image& albedo, const image::Image& picture, const oiio::ImageBuf pictureBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 1, oiio::TypeDesc::FLOAT), const_cast(picture.data())); oiio::ImageBuf albedoBuf(oiio::ImageSpec(picture.Width(), picture.Height(), 1, oiio::TypeDesc::FLOAT), albedo.data()); oiio::ImageBufAlgo::median_filter(albedoBuf, pictureBuf, albedoEstimationFilterSize, albedoEstimationFilterSize); - image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo, + image::EImageColorSpace::AUTO); } break; case EAlbedoEstimation::BLUR_FILTER: @@ -254,7 +257,8 @@ void initAlbedo(image::Image& albedo, const image::Image& picture, oiio::ImageBuf K; oiio::ImageBufAlgo::make_kernel(K, "gaussian", albedoEstimationFilterSize, albedoEstimationFilterSize); oiio::ImageBufAlgo::convolve(albedoBuf, pictureBuf, K); - image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo); + image::writeImage((fs::path(outputFolder) / (std::to_string(viewId) + "_albedo.jpg")).string(), albedo, + image::EImageColorSpace::AUTO); } break; } @@ -362,12 +366,12 @@ int main(int argc, char** argv) const std::string normalsPath = mvsUtils::getFileNameFromViewId(&mp, viewId, mvsUtils::EFileType::normalMap, 0); image::Image normals; - image::readImage(normalsPath, normals); + image::readImage(normalsPath, normals, image::EImageColorSpace::LINEAR); if(lightingColor == ELightingColor::Luminance) { image::Image albedo, picture; - image::readImage(picturePath, picture); + image::readImage(picturePath, picture, image::EImageColorSpace::LINEAR); initAlbedo(albedo, picture, albedoEstimationMethod, albedoEstimationFilterSize, outputFolder, viewId); @@ -376,7 +380,7 @@ int main(int argc, char** argv) else if(lightingColor == ELightingColor::RGB) { image::Image albedo, picture; - image::readImage(picturePath, picture); + image::readImage(picturePath, picture, image::EImageColorSpace::LINEAR); initAlbedo(albedo, picture, albedoEstimationMethod, albedoEstimationFilterSize, outputFolder, viewId); From 5fb772140b94d5471123d4d171d9a5b0f191e34f Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 15 May 2020 23:41:32 +0200 Subject: [PATCH 6/6] [robustEstimation] missing include for MSVC --- src/aliceVision/robustEstimation/randSampling.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aliceVision/robustEstimation/randSampling.hpp b/src/aliceVision/robustEstimation/randSampling.hpp index ac3c046141..182ab6c5f8 100644 --- a/src/aliceVision/robustEstimation/randSampling.hpp +++ b/src/aliceVision/robustEstimation/randSampling.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include