From cb558e909971e5d3e81d5346a2d8c178429fc930 Mon Sep 17 00:00:00 2001 From: Ashley Hauck Date: Wed, 26 Jul 2017 18:14:41 +0000 Subject: [PATCH 1/3] Move logic from the Makefile to either cibuild.sh or build/scripts Also update a couple docs to reflect more modern information. Docs are very out of date at this point, and a review pass should probably be done. --- Makefile | 69 --------------------------- build/scripts/obtain_dotnet.sh | 39 +++++++++++++++ build/scripts/tests.sh | 23 ++++++--- cibuild.sh | 65 +++++++++++++++---------- docs/infrastructure/cross-platform.md | 19 ++------ docs/infrastructure/unix-toolset.md | 29 ----------- 6 files changed, 100 insertions(+), 144 deletions(-) delete mode 100644 Makefile create mode 100644 build/scripts/obtain_dotnet.sh delete mode 100644 docs/infrastructure/unix-toolset.md diff --git a/Makefile b/Makefile deleted file mode 100644 index c00e72729f700..0000000000000 --- a/Makefile +++ /dev/null @@ -1,69 +0,0 @@ -SHELL = /usr/bin/env bash -OS_NAME = $(shell uname -s) -BUILD_CONFIGURATION = Debug -BINARIES_PATH = $(shell pwd)/Binaries -SRC_PATH = $(shell pwd)/src -BOOTSTRAP_PATH = $(BINARIES_PATH)/Bootstrap -BUILD_LOG_PATH = -HOME_DIR = $(shell cd ~ && pwd) -DOTNET_VERSION = 1.0.1 -DOTNET = $(BINARIES_PATH)/dotnet-cli/dotnet -TARGET_FX = netcoreapp1.1 - -MSBUILD_ADDITIONALARGS := /v:m /fl /fileloggerparameters:Verbosity=normal /p:Configuration=$(BUILD_CONFIGURATION) - -ifeq ($(OS_NAME),Linux) - RUNTIME_ID := $(shell . /etc/os-release && echo $$ID.$$VERSION_ID)-x64 -else ifeq ($(OS_NAME),Darwin) - RUNTIME_ID := osx.10.12-x64 -endif - -ifneq ($(BUILD_LOG_PATH),) - MSBUILD_ADDITIONALARGS := $(MSBUILD_ADDITIONALARGS) /fileloggerparameters:LogFile=$(BUILD_LOG_PATH) -endif - -ifeq ($(BOOTSTRAP),true) - MSBUILD_ARGS = $(MSBUILD_ADDITIONALARGS) /p:CscToolPath=$(BOOTSTRAP_PATH)/csc /p:CscToolExe=csc /p:VbcToolPath=$(BOOTSTRAP_PATH)/vbc /p:VbcToolExe=vbc -else - MSBUILD_ARGS = $(MSBUILD_ADDITIONALARGS) -endif - -BUILD_CMD = dotnet build $(MSBUILD_ARGS) - -.PHONY: all bootstrap test toolset - -all: restore - @export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \ - export HOME="$(HOME_DIR)" ; \ - $(BUILD_CMD) CrossPlatform.sln - -bootstrap: restore - export HOME="$(HOME_DIR)" ; \ - export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \ - $(BUILD_CMD) src/Compilers/CSharp/CscCore && \ - $(BUILD_CMD) src/Compilers/VisualBasic/VbcCore && \ - mkdir -p $(BOOTSTRAP_PATH)/csc && mkdir -p $(BOOTSTRAP_PATH)/vbc && \ - dotnet publish -c $(BUILD_CONFIGURATION) -r $(RUNTIME_ID) src/Compilers/CSharp/CscCore -o $(BOOTSTRAP_PATH)/csc && \ - dotnet publish -c $(BUILD_CONFIGURATION) -r $(RUNTIME_ID) src/Compilers/VisualBasic/VbcCore -o $(BOOTSTRAP_PATH)/vbc - rm -rf Binaries/$(BUILD_CONFIGURATION) - -test: - @export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \ - export HOME="$(HOME_DIR)" ; \ - dotnet publish -r $(RUNTIME_ID) src/Test/DeployCoreClrTestRuntime -o $(BINARIES_PATH)/$(BUILD_CONFIGURATION)/CoreClrTest -p:RoslynRuntimeIdentifier=$(RUNTIME_ID) && \ - build/scripts/tests.sh $(BUILD_CONFIGURATION) - -restore: $(DOTNET) - export PATH="$(BINARIES_PATH)/dotnet-cli:$(PATH)" ; \ - ./build/scripts/restore.sh - -$(DOTNET): - mkdir -p $(BINARIES_PATH) ; \ - pushd $(BINARIES_PATH) ; \ - curl -O https://raw.githubusercontent.com/dotnet/cli/rel/1.0.0/scripts/obtain/dotnet-install.sh && \ - chmod +x dotnet-install.sh && \ - ./dotnet-install.sh --version "$(DOTNET_VERSION)" --install-dir "$(BINARIES_PATH)/dotnet-cli" - - -clean: - @rm -rf Binaries diff --git a/build/scripts/obtain_dotnet.sh b/build/scripts/obtain_dotnet.sh new file mode 100644 index 0000000000000..7410d0811171f --- /dev/null +++ b/build/scripts/obtain_dotnet.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +# Source this script to ensure dotnet is installed and on the path. +# If the FORCE_DOWNLOAD environment variable is set to "true", the system's dotnet install is ignored, +# and dotnet is downloaded and installed locally. + +set -e +set -u + +# check if `dotnet` is already on the PATH +if command -v dotnet >/dev/null 2>&1 +then + if [[ "${FORCE_DOWNLOAD:-false}" != true ]] + then + exit 0 + fi +fi + +# This is a function to keep variable assignments out of the parent script (that is sourcing this file) +install_dotnet () { + # Download and install `dotnet` locally + local DOTNET_VERSION=1.0.1 + local THIS_DIR=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd) + local DOTNET_PATH=${THIS_DIR}/../../Binaries/dotnet-cli + + if [[ ! -x "${DOTNET_PATH}/dotnet" ]] + then + echo "Downloading and installing .NET CLI version ${DOTNET_VERSION} to ${DOTNET_PATH}" + curl https://raw.githubusercontent.com/dotnet/cli/rel/${DOTNET_VERSION}/scripts/obtain/dotnet-install.sh | \ + /usr/bin/env bash -s -- --version "${DOTNET_VERSION}" --install-dir "${DOTNET_PATH}" + else + echo "Skipping download of .NET CLI: Already installed at ${DOTNET_PATH}" + fi + + export PATH=${DOTNET_PATH}:${PATH} +} +install_dotnet diff --git a/build/scripts/tests.sh b/build/scripts/tests.sh index 2db9437ca5f4c..51b2a8d00ff1b 100755 --- a/build/scripts/tests.sh +++ b/build/scripts/tests.sh @@ -1,20 +1,29 @@ #!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. -BUILD_CONFIGURATION=$1 +set -e +set -u -# This function will update the PATH variable to put the desired -# version of Mono ahead of the system one. +BUILD_CONFIGURATION=${1:-Debug} -cd Binaries/$BUILD_CONFIGURATION/CoreClrTest +THIS_DIR=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd) +BINARIES_PATH=${THIS_DIR}/../../Binaries +SRC_PATH=${THIS_DIR}/../../src +TEST_DIR=${BINARIES_PATH}/${BUILD_CONFIGURATION}/CoreClrTest -chmod +x ./corerun +RUNTIME_ID=$(dotnet --info | awk '/RID:/{print $2;}') + +BUILD_ARGS="-c ${BUILD_CONFIGURATION} -r ${RUNTIME_ID} /consoleloggerparameters:Verbosity=minimal;summary /p:RoslynRuntimeIdentifier=${RUNTIME_ID}" +dotnet publish ${SRC_PATH}/Test/DeployCoreClrTestRuntime -o ${TEST_DIR} ${BUILD_ARGS} + +cd ${TEST_DIR} mkdir -p xUnitResults -./corerun ./xunit.console.netcore.exe *.UnitTests.dll -parallel all -xml xUnitResults/TestResults.xml +dotnet exec ./xunit.console.netcore.exe *.UnitTests.dll -parallel all -xml xUnitResults/TestResults.xml if [ $? -ne 0 ]; then echo Unit test failed exit 1 fi - diff --git a/cibuild.sh b/cibuild.sh index 7781ddb5f0c9f..c341c62233751 100755 --- a/cibuild.sh +++ b/cibuild.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + set -e +set -u usage() { @@ -9,23 +13,22 @@ usage() echo "Options" echo " --debug Build Debug (default)" echo " --release Build Release" + echo " --cleanrun Clean the project before building" echo " --skiptest Do not run tests" - echo " --skipcrossgen Do not crossgen the bootstrapped compiler" echo " --skipcommitprinting Do not print commit information" - echo " --nocache Force download of toolsets" } +THIS_DIR=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd) +BINARIES_PATH=${THIS_DIR}/Binaries +BOOTSTRAP_PATH=${BINARIES_PATH}/Bootstrap +SRC_PATH=${THIS_DIR}/src +BUILD_LOG_PATH=${BINARIES_PATH}/Build.log + BUILD_CONFIGURATION=Debug -USE_CACHE=true +CLEAN_RUN=false SKIP_TESTS=false -SKIP_CROSSGEN=false SKIP_COMMIT_PRINTING=false -MAKE="make" -if [[ $OSTYPE == *[Bb][Ss][Dd]* ]]; then - MAKE="gmake" -fi - # LTTNG is the logging infrastructure used by coreclr. Need this variable set # so it doesn't output warnings to the console. export LTTNG_HOME=$HOME @@ -51,18 +54,14 @@ do BUILD_CONFIGURATION=Release shift 1 ;; - --nocache) - USE_CACHE=false + --cleanrun) + CLEAN_RUN=true shift 1 ;; --skiptests) SKIP_TESTS=true shift 1 ;; - --skipcrossgen) - SKIP_CROSSGEN=true - shift 1 - ;; --skipcommitprinting) SKIP_COMMIT_PRINTING=true shift 1 @@ -74,8 +73,6 @@ do esac done -MAKE_ARGS="BUILD_CONFIGURATION=$BUILD_CONFIGURATION SKIP_CROSSGEN=$SKIP_CROSSGEN" - if [ "$CLEAN_RUN" == "true" ]; then echo Clean out the enlistment git clean -dxf . @@ -86,13 +83,33 @@ if [ "$SKIP_COMMIT_PRINTING" == "false" ]; then git show --no-patch --pretty=raw HEAD fi -echo Building Bootstrap -$MAKE bootstrap $MAKE_ARGS +# obtain_dotnet.sh puts the right dotnet on the PATH +FORCE_DOWNLOAD=true +source ${THIS_DIR}/build/scripts/obtain_dotnet.sh -echo Building CrossPlatform.sln -$MAKE all $MAKE_ARGS BOOTSTRAP=true BUILD_LOG_PATH=Binaries/Build.log +RUNTIME_ID=$(dotnet --info | awk '/RID:/{print $2;}') +echo "Using Runtime Identifier: ${RUNTIME_ID}" -if [ "$SKIP_TESTS" == "false" ]; then - $MAKE test $MAKE_ARGS -fi +RESTORE_ARGS="-r ${RUNTIME_ID} -v Minimal --disable-parallel" +echo "Restoring BaseToolset.csproj" +dotnet restore ${RESTORE_ARGS} ${THIS_DIR}/build/ToolsetPackages/BaseToolset.csproj +echo "Restoring CrossPlatform.sln" +dotnet restore ${RESTORE_ARGS} ${THIS_DIR}/CrossPlatform.sln + +BUILD_ARGS="-c ${BUILD_CONFIGURATION} -r ${RUNTIME_ID} /nologo /consoleloggerparameters:Verbosity=minimal;summary /filelogger /fileloggerparameters:Verbosity=normal;logFile=${BUILD_LOG_PATH} /p:RoslynRuntimeIdentifier=${RUNTIME_ID}" +echo "Building bootstrap CscCore" +dotnet publish ${SRC_PATH}/Compilers/CSharp/CscCore -o ${BOOTSTRAP_PATH}/csc ${BUILD_ARGS} +echo "Building bootstrap VbcCore" +dotnet publish ${SRC_PATH}/Compilers/VisualBasic/VbcCore -o ${BOOTSTRAP_PATH}/vbc ${BUILD_ARGS} +rm -rf ${BINARIES_PATH}/${BUILD_CONFIGURATION} +BUILD_ARGS+=" /p:CscToolPath=${BOOTSTRAP_PATH}/csc /p:CscToolExe=csc /p:VbcToolPath=${BOOTSTRAP_PATH}/vbc /p:VbcToolExe=vbc" + +echo "Building CrossPlatform.sln" +dotnet build ${THIS_DIR}/CrossPlatform.sln ${BUILD_ARGS} + +if [[ "${SKIP_TESTS}" == false ]] +then + echo "Running tests" + ${THIS_DIR}/build/scripts/tests.sh ${BUILD_CONFIGURATION} +fi diff --git a/docs/infrastructure/cross-platform.md b/docs/infrastructure/cross-platform.md index 9d5be313efd0a..c24fbff412a1b 100644 --- a/docs/infrastructure/cross-platform.md +++ b/docs/infrastructure/cross-platform.md @@ -4,26 +4,15 @@ Linux and Mac support for developing Roslyn is very much a work in progress. Not everything is supported at the moment and the steps detailed on this page will change very frequently. If this is an area you are interested in then please check back frequently for updates. -## Building using a pre-made toolset +## Building -Right now Roslyn builds on *nix using a mix of Mono and CoreCLR. Patching the right Mono version and acquiring all the tools -can be very difficult, so we've saved pre-built versions on Azure. +Build all cross-platform projects with: `dotnet build CrossPlatform.sln`. -Running `make` should download all these toolset binaries and kick off a build using MSBuild running on Mono. +If you do not have a system-wide `dotnet` install, you can obtain one with `./build/scripts/obtain_dotnet.sh`. This will install a compatible version of the CLI to `./Binaries/dotnet-cli` - add this to your PATH before trying to build `CrossPlatform.sln`. Alternatively, sourcing the script with `source ./build/scripts/obtain_dotnet.sh` will add it to your PATH for you. ## Using the compiler -After building there should be at least two versions of `csc.exe` in your output directory. - -The first is in the `Binaries/Debug` directory. This is the "full .NET framework" version. That means it expects to run on a -full .NET framework, like either the Windows .NET framework or Mono. You would run this like you run other mono programs, i.e. -`mono csc.exe`. - -The second copy is in the `Binaries/Debug/csccore` directory. This is a version running directly on CoreCLR -- no Mono necessary. -Just run `csc` in that directory. Note that this version includes a copy of CoreCLR in the output directory, so it is not portable. -The version of CoreCLR copied is specific to whatever machine you built with, so if you're running OS X, this will only run on OS X. -Similarly with Linux (and whatever distro you're using). - +After building, there will be a `csc` in the `Binaries/Debug/Exes/CscCore` directory. ### Known issues when running `csc.exe` diff --git a/docs/infrastructure/unix-toolset.md b/docs/infrastructure/unix-toolset.md deleted file mode 100644 index 7946c32097366..0000000000000 --- a/docs/infrastructure/unix-toolset.md +++ /dev/null @@ -1,29 +0,0 @@ -Building a new Unix toolset -==== -This document describes building a new toolset for use in Mac or Linux. -Because the toolsets contain various targets and reference assemblies that -only exist on Windows, the toolsets currently must be built on Windows. - -### Building Roslyn Toolset -The new *toolset name* will be chosen as one of the follownig: - -- Linux: roslyn.linux.`` -- Mac: roslyn.mac.`` - -The value of *version number* will simply be the one number higher than the current version number of the toolset. - -To build the toolset do the following: - -- If necessary, make modifications to the dependencies in the - `build/MSBuildToolset/project.json` file to bring in anything new. -- Run the `build/MSBuildToolset/build-toolset.ps1` file. -- The script produces two zip files in bin\Debug\netcoreapp1.0 subdirectory: - - Rename `osx.10.10-x64.zip` to roslyn.mac.`.zip` - - Rename `ubuntu.14.04-x64.zip` to roslyn.linux.`.zip` -- Upload the files to the Azure in the dotnetci storage account in the roslyn container: - -``` -azcopy /Pattern:*.zip /Source:build\MSBuildToolset\bin\Debug\netcoreapp1.0 /Dest:https://dotnetci.blob.core.windows.net/roslyn /DestKey:<> -``` - -- Send a PR to change [Makefile](https://github.com/dotnet/roslyn/blob/master/Makefile) to use the new toolset. From fb30b4d4a91107e2c666a3d06629564e23e9313b Mon Sep 17 00:00:00 2001 From: Ashley Hauck Date: Wed, 26 Jul 2017 18:24:00 +0000 Subject: [PATCH 2/3] Remove unused nocache flag from CI --- netci.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netci.groovy b/netci.groovy index 6be39abf538ab..32bd53b24de9f 100644 --- a/netci.groovy +++ b/netci.groovy @@ -103,7 +103,7 @@ commitPullList.each { isPr -> def myJob = job(jobName) { description("Ubuntu 14.04 tests") steps { - shell("./cibuild.sh --nocache --debug") + shell("./cibuild.sh --debug") } } @@ -120,7 +120,7 @@ commitPullList.each { isPr -> def myJob = job(jobName) { description("Ubuntu 16.04 tests") steps { - shell("./cibuild.sh --nocache --debug") + shell("./cibuild.sh --debug") } } @@ -137,7 +137,7 @@ commitPullList.each { isPr -> def myJob = job(jobName) { description("Mac tests") steps { - shell("./cibuild.sh --nocache --debug") + shell("./cibuild.sh --debug") } } From 6c5486cda2c8b70690f55c3ce23eadb12c346769 Mon Sep 17 00:00:00 2001 From: Ashley Hauck Date: Wed, 26 Jul 2017 19:49:25 +0000 Subject: [PATCH 3/3] Temporarily ignore --nocache flag Ignoring until the netci.groovy change gets merged, at which point another PR can be made to remove this ignore. --- cibuild.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cibuild.sh b/cibuild.sh index c341c62233751..a5fbecec5fb8a 100755 --- a/cibuild.sh +++ b/cibuild.sh @@ -66,6 +66,11 @@ do SKIP_COMMIT_PRINTING=true shift 1 ;; + --nocache) + # Temporarily ignore this argument until the change to netci.groovy gets merged. + # A follow-up PR will be made to remove this ignore. + shift 1 + ;; *) usage exit 1