trailofbits · scoders-tob · Sep 25, 2019 · Oct 16, 2019 · Sep 25, 2019 · Oct 21, 2019
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -20,8 +20,8 @@ jobs:
       vmImage: 'Ubuntu-16.04'
 
     container:
-      image: trailofbits/osquery:ubuntu-18.04-toolchain-v4
-      options: --privileged
+      image: trailofbits/osquery:ubuntu-18.04-toolchain-391938a2
+      options: --privileged --init
 
     timeoutInMinutes: 120
 
@@ -199,6 +199,7 @@ jobs:
     - script: |
         brew upgrade
         brew install ccache flex bison
+        pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery
         sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer
       displayName: "Install Homebrew and prerequisites"
       timeoutInMinutes: 20
@@ -328,6 +329,11 @@ jobs:
 
     - checkout: self
 
+    - powershell: |
+        $python3_path = ((Get-Item C:\hostedtoolcache\windows\Python\3*\x64) | Sort-Object -Descending)[0].FullName
+        & $python3_path\python -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32
+      displayName: Install tests prerequisites
+
     - powershell: |
         mkdir $(Build.BinariesDirectory)\build
       displayName: "Create build folder"

diff --git a/cmake/globals.cmake b/cmake/globals.cmake
@@ -19,7 +19,7 @@ endif()
 
 # This is the destination for the remotely imported Python modules, used when
 # setting up the PYTHONPATH folder
-set(PYTHON_PATH "${CMAKE_BINARY_DIR}/python_path")
+set(OSQUERY_PYTHON_PATH "${CMAKE_BINARY_DIR}/python_path")
 
 # TODO(alessandro): Add missing defines: PLATFORM_FREEBSD
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")

diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake
@@ -102,9 +102,6 @@ endfunction()
 function(findPythonExecutablePath)
   find_package(Python2 COMPONENTS Interpreter REQUIRED)
   find_package(Python3 COMPONENTS Interpreter REQUIRED)
-
-  set(EX_TOOL_PYTHON2_EXECUTABLE_PATH "${Python2_EXECUTABLE}" PARENT_SCOPE)
-  set(EX_TOOL_PYTHON3_EXECUTABLE_PATH "${Python3_EXECUTABLE}" PARENT_SCOPE)
 endfunction()
 
 function(generateBuildTimeSourceFile file_path content)
@@ -124,14 +121,23 @@ function(generateUnsupportedPlatformSourceFile)
   set(unsupported_platform_source_file "${source_file}" PARENT_SCOPE)
 endfunction()
 
-function(generateCopyFileTarget name type relative_file_paths destination)
-  set(source_base_path "${CMAKE_CURRENT_SOURCE_DIR}")
+function(generateCopyFileTarget name base_path type relative_file_paths destination)
+
+  if(base_path)
+    set(base_path "${base_path}/")
+  else()
+    set(base_path "${CMAKE_CURRENT_SOURCE_DIR}/")
+  endif()
 
   if(type STREQUAL "REGEX")
-    file(GLOB_RECURSE relative_file_paths RELATIVE "${source_base_path}" "${source_base_path}/${relative_file_paths}")
+    if(base_path)
+      file(GLOB_RECURSE relative_file_paths RELATIVE "${base_path}" "${base_path}${relative_file_paths}")
+    else()
+      file(GLOB_RECURSE relative_file_paths "${base_path}${relative_file_paths}")
+    endif()
   endif()
 
-  add_library("${name}" INTERFACE)
+  add_custom_target("${name}")
 
   foreach(file ${relative_file_paths})
     get_filename_component(intermediate_directory "${file}" DIRECTORY)
@@ -148,7 +154,16 @@ function(generateCopyFileTarget name type relative_file_paths destination)
     list(APPEND created_directories "${destination}/${directory}")
   endforeach()
 
-  add_custom_target("${name}_create_dirs" DEPENDS "${created_directories}")
+  list(APPEND "create_dirs_deps"
+    "${created_directories}"
+    "${destination}"
+  )
+
+  add_custom_target("${name}_create_dirs" DEPENDS "${create_dirs_deps}")
+  add_custom_command(
+    OUTPUT "${destination}"
+    COMMAND "${CMAKE_COMMAND}" -E make_directory "${destination}"
+  )
 
   foreach(file ${relative_file_paths})
 
@@ -160,16 +175,18 @@ function(generateCopyFileTarget name type relative_file_paths destination)
 
     add_custom_command(
       OUTPUT "${destination}/${file}"
-      COMMAND "${CMAKE_COMMAND}" -E copy "${source_base_path}/${file}" "${destination}/${file}"
+      COMMAND "${CMAKE_COMMAND}" -E copy "${base_path}${file}" "${destination}/${file}"
+      DEPENDS "${base_path}${file}"
     )
     list(APPEND copied_files "${destination}/${file}")
   endforeach()
 
-  add_custom_target("${name}_copy_files" DEPENDS "${name}_create_dirs" "${copied_files}")
+  add_custom_target("${name}_copy_files" DEPENDS "${copied_files}")
 
+  add_dependencies("${name}_copy_files" "${name}_create_dirs")
   add_dependencies("${name}" "${name}_copy_files")
 
-  set_target_properties("${name}" PROPERTIES INTERFACE_BINARY_DIR "${destination}")
+  set_target_properties("${name}" PROPERTIES FILES_DESTINATION_DIR "${destination}")
 endfunction()
 
 function(add_osquery_executable)
@@ -230,7 +247,6 @@ function(generateSpecialTargets)
   # Used to generate all the files necessary to have a complete view of the project in the IDE
   add_custom_target(prepare_for_ide)
 
-
   set(excluded_folders
     "libraries/cmake/source"
   )
@@ -241,12 +257,12 @@ function(generateSpecialTargets)
   endif()
 
   add_custom_target(format_check
-    COMMAND ${command_prefix} ${EX_TOOL_PYTHON2_EXECUTABLE_PATH} ${CMAKE_SOURCE_DIR}/tools/formatting/format-check.py --exclude-folders ${excluded_folders} origin/master
+    COMMAND ${command_prefix} "${Python2_EXECUTABLE}" ${CMAKE_SOURCE_DIR}/tools/formatting/format-check.py --exclude-folders ${excluded_folders} origin/master
     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
     VERBATIM
   )
   add_custom_target(format
-    COMMAND ${command_prefix} ${EX_TOOL_PYTHON2_EXECUTABLE_PATH} ${CMAKE_SOURCE_DIR}/tools/formatting/git-clang-format.py --exclude-folders ${excluded_folders} -f --style=file
+    COMMAND ${command_prefix} "${Python2_EXECUTABLE}" ${CMAKE_SOURCE_DIR}/tools/formatting/git-clang-format.py --exclude-folders ${excluded_folders} -f --style=file
     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
     VERBATIM
   )

diff --git a/docs/wiki/development/building.md b/docs/wiki/development/building.md
@@ -24,6 +24,10 @@ The root folder is assumed to be `/home/<user>`
 # Install the prerequisites
 sudo apt install --no-install-recommends git python python3 bison flex make
 
+# Optional: install python tests prerequisites
+sudo apt install --no-install-recommends python3-pip python3-setuptools python3-psutil python3-six python3-wheel
+pip3 install timeout_decorator thrift==0.11.0 osquery pexpect==3.3
+
 # Download and install the osquery toolchain
 wget https://github.com/osquery/osquery-toolchain/releases/download/1.0.0/osquery-toolchain-1.0.0.tar.xz
 sudo tar xvf osquery-toolchain-1.0.0.tar.xz -C /usr/local
@@ -55,6 +59,9 @@ Please ensure [Homebrew](https://brew.sh/) has been installed, first. Then do th
 # Install prerequisites
 xcode-select --install
 brew install git cmake python@2 python
+
+# Optional: install python tests prerequisites
+pip3 install setuptools pexpect==3.3 psutil timeout_decorator six thrift==0.11.0 osquery
 ```
 
 **Step 2: Download and build**
@@ -93,6 +100,14 @@ Note: It may be easier to install these prerequisites using [Chocolatey](https:/
 - [Strawberry Perl](http://strawberryperl.com/) for the OpenSSL formula. It is recommended to install it to the default destination path.
 - [7-Zip](https://www.7-zip.org/) if building the Chocolatey package.
 
+**Optional: Install python tests prerequisites**
+Python 3 is assumed to be installed in `C:\Program Files\Python37`
+
+```PowerShell
+# Using a PowerShell console
+& 'C:\Program Files\Python37\python.exe' -m pip install setuptools psutil timeout_decorator thrift==0.11.0 osquery pywin32
+```
+
 **Step 2: Download and build**
 
 ```PowerShell

diff --git a/libraries/cmake/facebook/modules/api.cmake b/libraries/cmake/facebook/modules/api.cmake
@@ -10,7 +10,7 @@ function(downloadRemoteFile identifier base_url file_name hash)
   set(destination "${CMAKE_CURRENT_BINARY_DIR}/${file_name}")
   set(url "${base_url}/${file_name}")
 
-  set(command_prefix "${EX_TOOL_PYTHON3_EXECUTABLE_PATH}")
+  set(command_prefix "${Python3_EXECUTABLE}")
 
   add_custom_command(
     OUTPUT "${destination}"
@@ -161,15 +161,17 @@ function(importThirdPartyHeaderOnlyLibrary library_type name version hash anchor
 endfunction()
 
 # Initializes the PYTHONPATH folder in the binary directory, used to run the codegen scripts
-function(initializePythonPathFolder)  
-  add_custom_command(
-    OUTPUT "${PYTHON_PATH}"
-    COMMAND "${CMAKE_COMMAND}" -E make_directory "${PYTHON_PATH}"
-    COMMENT "Initializing custom PYTHONPATH: ${PYTHON_PATH}"
-  )
-
-  add_custom_target(thirdparty_pythonpath DEPENDS "${PYTHON_PATH}")
-  add_custom_target(thirdparty_python_modules)
+function(initializePythonPathFolder)
+  if(NOT TARGET thirdparty_python_modules)
+    add_custom_command(
+      OUTPUT "${OSQUERY_PYTHON_PATH}"
+      COMMAND "${CMAKE_COMMAND}" -E make_directory "${OSQUERY_PYTHON_PATH}"
+      COMMENT "Initializing custom PYTHONPATH: ${OSQUERY_PYTHON_PATH}"
+    )
+
+    add_custom_target(thirdparty_pythonpath DEPENDS "${OSQUERY_PYTHON_PATH}")
+    add_custom_target(thirdparty_python_modules)
+  endif()
 endfunction()
 
 # Imports a remote Python module inside the PYTHONPATH folder (previously initialized
@@ -178,7 +180,7 @@ function(importRemotePythonModule identifier base_url file_name hash)
   set(target_name "thirdparty_pythonmodule_${identifier}")
   downloadRemoteFile("${target_name}" "${base_url}" "${file_name}" "${hash}")
 
-  extractLocalArchive("${target_name}" "${PYTHON_PATH}/${identifier}" "${downloadRemoteFile_destination}" "${PYTHON_PATH}")
+  extractLocalArchive("${target_name}" "${OSQUERY_PYTHON_PATH}/${identifier}" "${downloadRemoteFile_destination}" "${OSQUERY_PYTHON_PATH}")
   add_dependencies("${target_name}_extractor" thirdparty_pythonpath)
 
   add_osquery_library("${target_name}" INTERFACE)
@@ -199,15 +201,12 @@ function(importFacebookLibrary library_name)
 endfunction()
 
 # Make sure that globals.cmake and options.cmake have been included
-if("${PYTHON_PATH}" STREQUAL "")
-  message(FATAL_ERROR "The PYTHON_PATH variable was not found. Has globals.cmake been included?")
+if("${OSQUERY_PYTHON_PATH}" STREQUAL "")
+  message(FATAL_ERROR "The OSQUERY_PYTHON_PATH variable was not found. Has globals.cmake been included?")
 endif()
 
 if("${THIRD_PARTY_REPOSITORY_URL}" STREQUAL "")
   message(FATAL_ERROR "The THIRD_PARTY_REPOSITORY_URL variable was not found. Has options.cmake been included?")
 endif()
 
-if(NOT PYTHON_PATH_FOLDER_INITIALIZED)
-  initializePythonPathFolder()
-  set(PYTHON_PATH_FOLDER_INITIALIZED true)
-endif()
+initializePythonPathFolder()
diff --git a/libraries/cmake/facebook/thrift/CMakeLists.txt b/libraries/cmake/facebook/thrift/CMakeLists.txt
@@ -34,6 +34,7 @@ function(thrifMain)
   importThirdPartyBinaryLibrary("${name}" "${version}" "${hash}" "${anchor_file}" ${additional_libraries})
 
   target_link_libraries("thirdparty_${name}" INTERFACE thirdparty_boost)
+
 endfunction()
 
 thrifMain()
diff --git a/libraries/cmake/source/thrift/CMakeLists.txt b/libraries/cmake/source/thrift/CMakeLists.txt
@@ -109,6 +109,7 @@ function(thriftMain)
     "${library_root}/src"
     "${CMAKE_CURRENT_SOURCE_DIR}/config"
   )
+
 endfunction()
 
 thriftMain()
diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt
@@ -30,6 +30,8 @@ function(osqueryMain)
   add_subdirectory("ev2")
   add_subdirectory("experimental")
   add_subdirectory("system")
+  add_subdirectory("examples")
+  add_subdirectory("examples/extension_group_example")
 
   generateOsqueryHeaders()
   generateOsqueryd()
@@ -64,6 +66,18 @@ function(generateOsqueryd)
     osquery_cxx_settings
     osquery_main
   )
+
+  set(osquery_ext "")
+  if(PLATFORM_WINDOWS)
+    set(osquery_ext ".exe")
+  endif()
+
+  add_custom_target(create_osqueryi ALL DEPENDS osqueryi${osquery_ext})
+  add_custom_command(OUTPUT osqueryi${osquery_ext}
+    COMMAND "${CMAKE_COMMAND}" -E create_symlink osqueryd${osquery_ext} osqueryi${osquery_ext}
+    WORKING_DIRECTORY "$<TARGET_FILE_DIR:osqueryd>"
+  )
+  add_dependencies(create_osqueryi osqueryd)
 endfunction()
 
 osqueryMain()
diff --git a/osquery/config/tests/BUCK b/osquery/config/tests/BUCK
@@ -14,7 +14,7 @@ osquery_cxx_test(
     ],
     env = {
         "TEST_CONF_FILES_DIR": "$(location {})".format(
-            osquery_target("tools/tests:conf_files"),
+            osquery_target("tools/tests/configs:conf_files"),
         ),
     },
     visibility = ["PUBLIC"],
@@ -42,7 +42,7 @@ osquery_cxx_test(
     ],
     env = {
         "TEST_CONF_FILES_DIR": "$(location {})".format(
-            osquery_target("tools/tests:conf_files"),
+            osquery_target("tools/tests/configs:conf_files"),
         ),
     },
     visibility = ["PUBLIC"],

diff --git a/osquery/config/tests/CMakeLists.txt b/osquery/config/tests/CMakeLists.txt
@@ -45,10 +45,11 @@ function(generateOsqueryConfigTestsTest)
     plugins_database_ephemeral
     specs_tables
     osquery_config_tests_testutils
-    osquery_tools_tests_conffiles
     thirdparty_googletest
   )
 
+  add_dependencies(osquery_config_tests-test osquery_tools_tests_conffiles)
+
 endfunction()
 
 function(generateOsqueryConfigTestsPacksTest)
@@ -69,10 +70,11 @@ function(generateOsqueryConfigTestsPacksTest)
     plugins_database_ephemeral
     specs_tables
     osquery_config_tests_testutils
-    osquery_tools_tests_conffiles
     thirdparty_googletest
   )
 
+  add_dependencies(osquery_config_tests_packs-test osquery_tools_tests_conffiles)
+
 endfunction()
 
 osqueryConfigTestsMain()
diff --git a/osquery/distributed/tests/BUCK b/osquery/distributed/tests/BUCK
@@ -14,7 +14,7 @@ osquery_cxx_test(
     ],
     env = {
         "TEST_CONF_FILES_DIR": "$(location {})".format(
-            osquery_target("tools/tests:test_files"),
+            osquery_target("tools/tests/configs:test_files"),
         ),
     },
     visibility = ["PUBLIC"],

diff --git a/osquery/examples/CMakeLists.txt b/osquery/examples/CMakeLists.txt
@@ -0,0 +1,11 @@
+include(${CMAKE_SOURCE_DIR}/external/cmake/cmakelibs.cmake)
+
+function(osqueryExtensionsExampleMain)
+    addOsqueryExtension(example_extension example_extension.cpp)
+    target_link_libraries(example_extension PRIVATE
+        osquery_sdk_pluginsdk
+        osquery_extensions_implthrift
+    )
+endfunction()
+
+osqueryExtensionsExampleMain()
diff --git a/osquery/examples/example_extension.cpp b/osquery/examples/example_extension.cpp
@@ -6,7 +6,8 @@
  *  the LICENSE file found in the root directory of this source tree.
  */
 
-#include <osquery/sdk.h>
+#include <osquery/sdk/sdk.h>
+#include <osquery/sql/dynamic_table_row.h>
 #include <osquery/system.h>
 
 using namespace osquery;
@@ -72,10 +73,10 @@ class ComplexExampleTable : public TablePlugin {
     auto r = make_table_row();
 
     // Use the basic 'force' flag to check implicit SQL usage.
-    auto flags =
-        SQL("select default_value from osquery_flags where name = 'force'");
-    if (flags.rows().size() > 0) {
-      r["flag_test"] = flags.rows().back().at("default_value");
+    auto flags = SQL::selectFrom(
+        {"default_value"}, "osquery_flags", "name", EQUALS, "force");
+    if (flags.size() > 0) {
+      r["flag_test"] = flags[0]["default_value"];
     }
 
     std::string content;