Add transform support for LightGBM by open source FreeForm2 library

.ci/test.sh

@@ -80,7 +80,10 @@ if [[ $TASK == "lint" ]]; then
     echo "Linting R code"
     Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit -1
     echo "Linting C++ code"
-    cpplint --filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length --recursive ./src ./include ./R-package ./swig ./tests || exit -1
+    cpplint --filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length --exclude=./src/transform --recursive ./src ./include ./R-package ./swig ./tests|| exit -1
+    # Make a separate lint C++ check for transform codes, because they use non-const references, which is allowed by Google C++ Style Guide now.
+    # See https://github.com/cpplint/cpplint/issues/148.
+    cpplint --filter=-build/c++11,-build/include_subdir,-build/header_guard,-whitespace/line_length,-runtime/references --recursive ./src/transform || exit -1
     cmake_files=$(find . -name CMakeLists.txt -o -path "*/cmake/*.cmake")
     cmakelint --linelength=120 ${cmake_files} || true
     exit 0

src/transform/CMakeLists.txt

@@ -0,0 +1,96 @@
+cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR)
+
+project(transform)
+
+# following lib paths are required by linking directories not compiling.
+set(LLVM_LIB
+    LLVMAnalysis
+    LLVMAsmParser
+    LLVMAsmPrinter
+    LLVMBitReader
+    LLVMBitWriter
+    LLVMCodeGen
+    LLVMCore
+    LLVMDebugInfo
+    LLVMExecutionEngine
+    LLVMInstCombine
+    LLVMInstrumentation
+    LLVMInterpreter
+    LLVMipa
+    LLVMipo
+    LLVMIRReader
+    LLVMJIT
+    LLVMLineEditor
+    LLVMLinker
+    LLVMLTO
+    LLVMMC
+    LLVMMCAnalysis
+    LLVMMCDisassembler
+    LLVMMCJIT
+    LLVMMCParser
+    LLVMObjCARCOpts
+    LLVMObject
+    LLVMOption
+    LLVMProfileData
+    LLVMRuntimeDyld
+    LLVMScalarOpts
+    LLVMSelectionDAG
+    LLVMSupport
+    LLVMTarget
+    LLVMTransformUtils
+    LLVMVectorize
+    LLVMX86AsmParser
+    LLVMX86AsmPrinter
+    LLVMX86CodeGen
+    LLVMX86Desc
+    LLVMX86Disassembler
+    LLVMX86Info
+    LLVMX86Utils
+    )
+
+set(BOOST_LIB
+    boost_atomic
+    boost_chrono
+    boost_context
+    boost_coroutine
+    boost_date_time
+    boost_exception
+    boost_filesystem
+    boost_graph
+    boost_graph_parallel
+    boost_iostreams
+    boost_locale
+    boost_log
+    boost_log_setup
+    boost_math_c99
+    boost_math_c99f
+    boost_math_c99l
+    boost_math_tr1
+    boost_math_tr1f
+    boost_math_tr1l
+    boost_mpi
+    boost_prg_exec_monitor
+    boost_program_options
+    boost_random
+    boost_regex
+    boost_serialization
+    boost_system
+    boost_test_exec_monitor
+    boost_thread
+    boost_timer
+    boost_unit_test_framework
+    boost_wave
+    boost_wserialization
+    )
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+if(USE_DEBUG)
+    SET(CMAKE_BUILD_TYPE "Debug")
+    SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb")
+    SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall")
+endif()
+
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/NeuralTree.Library)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/DynamicRank.FreeForm.Library)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/NeuralTreeEvaluator.Library)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/TransformProcessor)

src/transform/DynamicRank.FreeForm.Library/CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR)
+
+project(DynamicRank.FreeForm.Library)
+
+set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive")
+
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Shared)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Expression)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Parse/SExpression/libs)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Transform)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Backend/llvm)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/External)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test)
+
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/inc/
+        DESTINATION include
+        FILES_MATCHING PATTERN "*.h"
+        )

src/transform/DynamicRank.FreeForm.Library/inc/FreeForm2.h

@@ -0,0 +1,18 @@
+/*!
+ * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See LICENSE file in the project root for license information.
+ */
+// FreeForm2.h declares the main interface to the FreeForm2 expression
+// evaluator, used for feature evaluation in the neural net/tree ensemble
+// evaluator.
+
+#pragma once
+
+#ifndef FREEFORM2_FREEFORM2_H
+#define FREEFORM2_FREEFORM2_H
+
+#include "FreeForm2Executable.h"
+#include "FreeForm2Compiler.h"
+#include "FreeForm2Program.h"
+
+#endif

src/transform/DynamicRank.FreeForm.Library/inc/FreeForm2Assert.h

@@ -0,0 +1,41 @@
+/*!
+ * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See LICENSE file in the project root for
+ * license information.
+ */
+#pragma once
+
+#ifndef FREEFORM2_ASSERT_H
+#define FREEFORM2_ASSERT_H
+
+namespace FreeForm2 {
+// Assert a condition, providing file and line of the assertion call.
+// Note that this function throws on failure, rather than aborting (like
+// standard c assert).
+void ThrowAssert(bool p_condition, const char *p_file, unsigned int p_line);
+
+// Assert a condition, providing the expression, file and line of the
+// assertion call.
+// Note that this function throws on failure, rather than aborting (like
+// standard c assert).
+void ThrowAssert(bool p_condition, const char *p_expression, const char *p_file,
+                 unsigned int p_line);
+
+// Assert that this function call should not be reached during normal
+// program execution.  Note that this function throws on failure, rather
+// than aborting.
+// __declspec(noreturn)
+void Unreachable(const char *p_file, unsigned int p_line);
+};  // namespace FreeForm2
+
+// Macros for asserting.
+// Call the regular FreeForm2::ThrowAssert function with macro-generated
+// parameters
+#define FF2_ASSERT(cond) \
+  FreeForm2::ThrowAssert((cond), #cond, __FILE__, __LINE__)
+
+// Call the regular FreeForm2::Unreachable function with macro-generated
+// parameters
+#define FF2_UNREACHABLE() FreeForm2::Unreachable(__FILE__, __LINE__)
+
+#endif

src/transform/DynamicRank.FreeForm.Library/inc/FreeForm2Compiler.h

@@ -0,0 +1,61 @@
+/*!
+ * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See LICENSE file in the project root for
+ * license information.
+ */
+#pragma once
+
+#ifndef FREEFORM2_INC_COMPILER_H
+#define FREEFORM2_INC_COMPILER_H
+
+#include <boost/noncopyable.hpp>
+#include <boost/scoped_ptr.hpp>
+#include <memory>
+#include <string>
+
+namespace DynamicRank {
+class IFeatureMap;
+}
+
+namespace FreeForm2 {
+class CompilerImpl;
+class ExternalDataManager;
+class Program;
+
+// This class contains the results of the compilation process. Since the
+// result of the compilation process is backend-dependent, this class
+// should be downcast to the backend result type according to the compiler
+// used.
+class CompilerResults : boost::noncopyable {
+ public:
+  virtual ~CompilerResults();
+};
+
+// A compiler compiles multiple programs using the same set of compiler
+// resources, which amortises costs across different programs.
+class Compiler : boost::noncopyable {
+ public:
+  explicit Compiler(std::auto_ptr<CompilerImpl> p_impl);
+
+  ~Compiler();
+
+  // Compile the program producing a backend-dependent results obejct.
+  // This method, optionally producing debug output on stderr.
+  std::unique_ptr<CompilerResults> Compile(const Program &p_program,
+                                           bool p_debugOutput);
+
+  // Default optimization level, used whenever the level is not explicitly
+  // specified.
+  static const unsigned int c_defaultOptimizationLevel = 0;
+
+ private:
+  // Pointer to implementation class (pimpl idiom).
+  boost::scoped_ptr<CompilerImpl> m_impl;
+};
+}  // namespace FreeForm2
+
+// This is used to prevent a breaking API change (moving compiler factory into
+// a separate header).
+#include "FreeForm2CompilerFactory.h"
+
+#endif

src/transform/DynamicRank.FreeForm.Library/inc/FreeForm2CompilerFactory.h

@@ -0,0 +1,74 @@
+/*!
+ * Copyright (c) 2021 Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See LICENSE file in the project root for
+ * license information.
+ */
+#pragma once
+
+#ifndef FREEFORM2_COMPILER_FACTORY_H
+#define FREEFORM2_COMPILER_FACTORY_H
+
+#include <memory>
+#include <string>
+
+namespace FreeForm2 {
+namespace Cpp {
+class ExternResolver;
+}
+
+class Compiler;
+
+// This class contains methods to instantiate the various types of
+// compilers.
+class CompilerFactory {
+ public:
+  enum DestinationFunctionType {
+    // typedef float (*DirectEvalFun)(StreamFeatureInput*, const FeatureType[],
+    // OutputType[]).
+    SingleDocumentEvaluation,
+
+    // typedef float (*AggregatedEvalFun)(StreamFeatureInput*, const
+    // FeatureType[][], UInt32, UInt32, OutputType[]).
+    DocumentSetEvaluation,
+  };
+
+  // Create a compiler which takes a program and compiles it into an
+  // ExecutableCompilerResults object. p_optimizationLevel describes the
+  // degree of optimization to perform on the code with an integer,
+  // analagous to 'gcc -O p_optimizationLevel'.
+  static std::unique_ptr<Compiler> CreateExecutableCompiler(
+      unsigned int p_optimizationLevel,
+      DestinationFunctionType p_destinationFunctionType =
+          SingleDocumentEvaluation);
+
+  // This method creates a Compiler object to compile a program to a
+  // C++/IFM target. The results of this compiler are defined in
+  // FreeForm2CppCompiler.h.
+  static std::unique_ptr<Compiler> CreateCppIFMCompiler(
+      const Cpp::ExternResolver &p_resolver);
+
+  // This method creates a Compiler object to compile a program to a
+  // C++/Barramundi target. The results of this compiler are defined
+  // in FreeForm2CppCompiler.h.
+  static std::unique_ptr<Compiler> CreateCppBarramundiCompiler(
+      const Cpp::ExternResolver &p_resolver, const std::string &p_metadataPath);
+
+  // This method creates a Compiler object to compile a program to a
+  // C++/Barramundi target with debugging instrumentation present in the
+  // program. The printf command is the name of a printf-style function
+  // for use with debugging statements. The results of this compiler are
+  // defined in FreeForm2CppCompiler.h.
+  static std::unique_ptr<Compiler> CreateDebuggingCppBarramundiCompiler(
+      const Cpp::ExternResolver &p_resolver, const std::string &p_metadataPath,
+      const std::string &p_printfCommand);
+
+  // This method creates a Compiler object to compile a program to a
+  // C++/FPGA target. The results of this compiler are defined
+  // in FreeForm2CppCompiler.h.
+  static std::unique_ptr<Compiler> CreateCppFpgaCompiler(
+      const Cpp::ExternResolver &p_resolver,
+      const std::string &p_outputMappingPath, const std::string &p_msdlPath);
+};
+}  // namespace FreeForm2
+
+#endif