Merge branch develop into main [2023_09_07] (#175)

* Agent: Support for building an Ubuntu Core snap. * Support Ubuntu 22.04 and Debian 11 builds. * Agent: Replace boost::filesystem usage with std::filesystem. * SDK (Windows): Preview additional methods for partial downloads, streaming downloads, and download properties. * bootstrap script determines OS and version on its own. * Fix issue of non-existent /include path during bootstrap and/or build.
microsoft · Sep 7, 2023 · d71ade6 · d71ade6
2 parents df639ab + 5c090ee
commit d71ade6
Show file tree

Hide file tree

Showing 91 changed files with 2,733 additions and 1,189 deletions.
diff --git a/.gitignore b/.gitignore
@@ -56,3 +56,7 @@ _deps
 *_CPack_Packages/
 *.deb
 out/
+
+# Ubuntu Snap files
+*.snap
+snap/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,10 +15,7 @@ option (DO_INCLUDE_PLUGINS "Build subproject plugins" OFF)
 option (DO_INCLUDE_SDK "Build subproject sdk-cpp" OFF)
 
 option (DO_BUILD_TESTS "Set DO_BUILD_TESTS to OFF to skip building tests." ON)
-
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
+option (DO_BUILD_FOR_SNAP "Enable DO Snap build option" OFF)
 
 # Get verbose output from cmake generation and build steps
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -61,6 +58,16 @@ else()
     message(FATAL_ERROR "Unknown platform for client")
 endif()
 
+# C++17 is required to use std::filesystem but Edge builds on Windows is not ready to move up yet.
+# Luckily, we do not need std::filesystem on Windows.
+if (DO_INCLUDE_SDK AND DO_PLATFORM_WINDOWS)
+    set(CMAKE_CXX_STANDARD 14)
+else ()
+    set(CMAKE_CXX_STANDARD 17)
+endif ()
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
 # PIC (Position Independent Code) ensures .a files can be linked to executables that have PIE enabled
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
@@ -82,7 +89,14 @@ if (DO_PLATFORM_LINUX)
     # (Enable --trace-expand in cmake and look at lines from boost_headers-config.cmake and its callers.)
     # Workaround: force cmake to look in "/usr" since the /lib and /bin paths are either
     # not relevant for our usage or they are symlinks to /usr/lib and /usr/bin.
-    set(CMAKE_PREFIX_PATH "/usr")
+    if (IS_SYMLINK "/lib" AND NOT EXISTS "/include")
+        if (CMAKE_PREFIX_PATH)
+            message (WARNING "/lib is a symlink and /include does not exist. CMAKE_PREFIX_PATH is already set, not overriding it.")
+        else ()
+            message (WARNING "/lib is a symlink and /include does not exist. Updating CMAKE_PREFIX_PATH to /usr.")
+            set(CMAKE_PREFIX_PATH "/usr")
+        endif ()
+    endif ()
 endif (DO_PLATFORM_LINUX)
 
 if (DO_PLATFORM_WINDOWS AND DO_INCLUDE_SDK)

diff --git a/README.md b/README.md
@@ -102,7 +102,7 @@ This will handle installing to the appropriate paths, and also the necessary set
 > cd /tmp/build-deliveryoptimization-agent/linux-debug/
 > sudo apt-get install ./deliveryoptimization-agent*.deb
 > cd /tmp/build-deliveryoptimization-plugin-apt/linux-debug/
-> sudo apt get install ./deliveryoptimization-plugin-apt*.deb
+> sudo apt-get install ./deliveryoptimization-plugin-apt*.deb
 ```
 
 2. If you build and install using cmake, or through some other custom means, be sure to setup the DO user/groups correctly in your installation.

diff --git a/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml b/azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml
@@ -33,13 +33,31 @@ jobs:
 
 - template: ../templates/do-docker-jobs.yml
   parameters:
-    targetOsArch: 'debian10_arm64'
+    targetOsArch: 'debian10_arm64' # azure pipelines does not support '.' in display names
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
 - template: ../templates/do-docker-jobs.yml
   parameters:
-    targetOsArch: 'debian10_x64'
+    targetOsArch: 'debian10_x64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm32' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_x64' # azure pipelines does not support '.' in display names
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
@@ -61,3 +79,14 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_arm64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_x64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
diff --git a/azure-pipelines/build/linux/du/docker/dopapt-docker.yml b/azure-pipelines/build/linux/du/docker/dopapt-docker.yml
@@ -43,6 +43,24 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dopapt-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm32'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'ubuntu1804_arm64'
@@ -61,3 +79,14 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dopapt-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
diff --git a/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml b/azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml
@@ -43,6 +43,24 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm32'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'ubuntu1804_arm64'
@@ -60,3 +78,15 @@ jobs:
     targetOsArch: 'ubuntu2004_x64'
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
diff --git a/azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml b/azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml
@@ -14,14 +14,14 @@ steps:
   displayName: Login to ACR
   inputs:
     command: login
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR   
     repository: $(parameters.targetOsArch)
 
 - task: Docker@2
   displayName: Pull latest build image
   inputs:
     command: pull
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}'
 
 - task: CmdLine@2

diff --git a/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml b/azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml
@@ -24,7 +24,7 @@ steps:
   inputs:
     targetType: 'filePath'
     filePath: 'build/scripts/check_binary_size.sh'
-    arguments: '1251928 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent'
+    arguments: '523672 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent'
   displayName: 'Limit binary size increase'
 
 - task: CmdLine@2

diff --git a/azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml b/azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml
@@ -14,14 +14,14 @@ steps:
   displayName: Login to ACR
   inputs:
     command: login
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     repository: $(parameters.targetOsArch)
 
 - task: Docker@2
   displayName: Pull latest build image
   inputs:
     command: pull
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}'
 
 - task: CmdLine@2

diff --git a/azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml b/azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml
@@ -14,14 +14,14 @@ steps:
   displayName: Login to ACR
   inputs:
     command: login
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     repository: $(parameters.targetOsArch)
 
 - task: Docker@2
   displayName: Pull latest build image
   inputs:
     command: pull
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}'
 
 - task: CmdLine@2

diff --git a/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml b/azure-pipelines/build/linux/du/templates/dosdkcpp-native-steps.yml
@@ -38,7 +38,7 @@ steps:
   inputs:
     targetType: 'filePath'
     filePath: 'build/scripts/check_binary_size.sh'
-    arguments: '443632 /tmp/build-deliveryoptimization-sdk/linux-${{parameters.config}}/sdk-cpp/libdeliveryoptimization.so.*.*.*'
+    arguments: '596080 /tmp/build-deliveryoptimization-sdk/linux-${{parameters.config}}/sdk-cpp/libdeliveryoptimization.so.*.*.?'
   displayName: 'Limit binary size increase'
 
 - task: CmdLine@2

diff --git a/azure-pipelines/build/windows/dosdkcpp.yml b/azure-pipelines/build/windows/dosdkcpp.yml
@@ -32,7 +32,7 @@ pr:
       - sdk-cpp/build/cleanup-install.sh
 
 pool:
-  vmImage: "windows-2019"
+  vmImage: "windows-2022"
 
 jobs:
 - job: Debug

diff --git a/azure-pipelines/publishing/github-release.yml b/azure-pipelines/publishing/github-release.yml
@@ -60,6 +60,14 @@ stages:
     parameters:
       targetOsArch: 'ubuntu2004_arm64'
       imageVersion: ${{variables.containerImageVersion}}
+  - template: templates/release-docker-build-steps.yml
+    parameters:
+      targetOsArch: 'ubuntu2204_x64'
+      imageVersion: ${{variables.containerImageVersion}}
+  - template: templates/release-docker-build-steps.yml
+    parameters:
+      targetOsArch: 'ubuntu2204_arm64'
+      imageVersion: ${{variables.containerImageVersion}}
   - template: templates/release-docker-build-steps.yml
     parameters:
       targetOsArch: 'debian10_arm32'
@@ -72,6 +80,18 @@ stages:
     parameters:
       targetOsArch: 'debian10_x64'
       imageVersion: ${{variables.containerImageVersion}}
+  - template: templates/release-docker-build-steps.yml
+    parameters:
+      targetOsArch: 'debian11_arm32'
+      imageVersion: ${{variables.containerImageVersion}}
+  - template: templates/release-docker-build-steps.yml
+    parameters:
+      targetOsArch: 'debian11_arm64'
+      imageVersion: ${{variables.containerImageVersion}}
+  - template: templates/release-docker-build-steps.yml
+    parameters:
+      targetOsArch: 'debian11_x64'
+      imageVersion: ${{variables.containerImageVersion}}
 
 - stage: release
   condition: succeeded()

diff --git a/build/build-snaps.sh b/build/build-snaps.sh
@@ -0,0 +1,34 @@
+#! /bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+set -e
+
+if [ $1 == "agent" -o $1 == "sdk-tests" ]
+then
+    echo "Building $1 snap..."
+
+    if [ ! -d "./snap" ]
+    then
+        mkdir ./snap
+    fi
+
+    if [ $1 == "agent" ]
+    then
+        cp ./snapcraft-options/snapcraft-agent.yaml ./snap/snapcraft.yaml
+
+    else
+        cp ./snapcraft-options/snapcraft-sdk.yaml ./snap/snapcraft.yaml
+    fi
+
+    if [ $2 -a $2  == "arm64" ]
+    then
+        snapcraft --target-arch=arm64 --destructive-mode --enable-experimental-target-arch
+    else
+        snapcraft
+    fi
+
+else
+    echo "$1 is not a valid snap option, valid options are 'agent' or 'sdk-tests'"
+fi