From 4f4f0e989468bea784a3c896bd0394d4b918981c Mon Sep 17 00:00:00 2001 From: Egor Chesakov Date: Wed, 24 Oct 2018 10:58:11 -0700 Subject: [PATCH] [release/2.2] Remove the Smarty dependency and move Windows ARM to Helix queue Port (partially) the following changes: * Build xunit wrappers the same way on windows and unix (#18695) * Work around cmd command length limit in xunit Exec task (#19095) * Update vc-runtime package to support ARM and ARM64 with current builds (#19265) * build-test - fix TestWrapper CS warnings (#19180) * Clean up build.cmd/build-test.cmd/runtest.cmd (#19291) * Use runtest.py to run tests for all platforms (#19213) * Updating runtest.py so that it works with Python 3 (#19768) * Fix build-test.sh wrapper build (#19779) * Move ARM64 Windows boxen to be Helix-provisioned (#20204) * Runtest.py on Windows Arm(64) (#20227) * Use runtest.cmd for arm(64) windows testing (#20301) * Do not restore or initialize buildtools for x86/arm64 (#20370) * Add hack for arm64/x86 to skip build tools restore (#20390) Update Xunit to 2.4.1 prerelease version Use the latest version (dbf0bf1) of tests/runtest.py --- build-test.cmd | 42 +- .../Dotnet-CoreClr-Trusted-BuildTests.json | 2 +- dependencies.props | 9 +- dir.common.props | 36 + dir.props | 14 +- init-tools.cmd | 35 +- netci.groovy | 246 +- run.sh | 2 +- tests/build.proj | 3 +- tests/dir.common.props | 28 + tests/dir.props | 5 +- tests/dir.sdkbuild.props | 14 + tests/helixprep.proj | 22 +- tests/issues.targets | 184 +- tests/override.targets | 4 - tests/publishdependency.targets | 4 +- tests/runtest.cmd | 141 +- tests/runtest.proj | 83 +- tests/runtest.py | 2034 +++++++++++++++++ tests/scripts/scripts.csproj | 4 +- tests/src/CLRTest.GC.targets | 5 +- .../Coreclr.TestWrapper.csproj | 38 +- .../CoreclrTestWrapperLib.cs | 10 - .../Desktop.Coreclr.TestWrapper.csproj | 45 - tests/src/Common/external/external.depproj | 21 +- tests/src/Common/vc_runtime/vc_runtime.csproj | 8 +- .../CodeQuality/Layout/SearchLoops.cs | 4 +- .../CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs | 2 +- .../benchmark+roslyn/benchmark+roslyn.csproj | 4 +- .../benchmark+serialize.csproj | 4 +- .../src/JIT/config/benchmark/benchmark.csproj | 4 +- .../TestWrappersConfig.csproj | 38 - tests/src/dir.common.props | 63 - tests/src/dir.props | 51 +- tests/src/dirs.proj | 3 +- tests/src/performance/performance.csproj | 4 +- tests/tests.targets | 45 +- tests/xunitwrapper.targets | 61 - 38 files changed, 2734 insertions(+), 588 deletions(-) create mode 100644 dir.common.props create mode 100644 tests/dir.common.props create mode 100644 tests/dir.sdkbuild.props create mode 100755 tests/runtest.py delete mode 100644 tests/src/Common/Desktop.Coreclr.TestWrapper/Desktop.Coreclr.TestWrapper.csproj delete mode 100644 tests/src/TestWrappersConfig/TestWrappersConfig.csproj delete mode 100644 tests/src/dir.common.props delete mode 100644 tests/xunitwrapper.targets diff --git a/build-test.cmd b/build-test.cmd index e65cbcacff97..8871afc8fa7b 100644 --- a/build-test.cmd +++ b/build-test.cmd @@ -46,6 +46,7 @@ set processedArgs= set __unprocessedBuildArgs= set __RunArgs= set __BuildAgainstPackagesArg= +set __BuildAgainstPackagesMsbuildArg= set __RuntimeId= set __ZipTests= set __TargetsWindows=1 @@ -56,6 +57,8 @@ set __DoCrossgen= @REM appears to CMD parsing as "-priority 1". Handle -priority specially to avoid problems, @REM and allow the "-priority=1" syntax. set __Priority=0 +set __PriorityArg= +set __PassThroughArg= :Arg_Loop if "%1" == "" goto ArgsDone @@ -75,13 +78,14 @@ if /i "%1" == "checked" (set __BuildType=Checked&set processedArgs if /i "%1" == "skipmanaged" (set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "toolset_dir" (set __ToolsetDir=%2&set __PassThroughArgs=%__PassThroughArgs% %2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) -if /i "%1" == "buildagainstpackages" (set __ZipTests=1&set __BuildAgainstPackagesArg=-BuildTestsAgainstPackages&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "buildagainstpackages" (set __ZipTests=1&set __BuildAgainstPackagesArg=-BuildTestsAgainstPackages&set __BuildAgainstPackagesMsbuildArg=/p:BuildTestsAgainstPackages=true&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "ziptests" (set __ZipTests=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgen" (set __DoCrossgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "runtimeid" (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "targetsNonWindows" (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "Exclude" (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "-priority" (set __Priority=%2&shift&set processedArgs=!processedArgs! %1=%2&shift&goto Arg_Loop) +if /i "%1" == "--" (set __PassThroughArg=%1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if [!processedArgs!]==[] ( set __UnprocessedBuildArgs=%__args% @@ -95,7 +99,7 @@ if [!processedArgs!]==[] ( :ArgsDone @REM Special handling for -priority=N argument. -if %__Priority% GTR 0 (set "__UnprocessedBuildArgs=!__UnprocessedBuildArgs! -priority=%__Priority%") +if %__Priority% GTR 0 (set "__PriorityArg=-priority=%__Priority%") if defined __BuildAgainstPackagesArg ( if not defined __RuntimeID ( @@ -107,6 +111,8 @@ if defined __BuildAgainstPackagesArg ( @if defined _echo @echo on set __RunArgs=-BuildOS=%__BuildOS% -BuildType=%__BuildType% -BuildArch=%__BuildArch% +REM As we move from buildtools to arcade, __RunArgs should be replaced with __msbuildArgs +set __msbuildArgs=/p:__BuildOS=%__BuildOS% /p:__BuildType=%__BuildType% /p:__BuildArch=%__BuildArch% if defined __ToolsetDir ( rem arm64 builds currently use private toolset which has not been released yet @@ -150,6 +156,12 @@ REM === REM ========================================================================================= call "%__ProjectDir%\init-tools.cmd" @if defined _echo @echo on +set "__ToolsDir=%__ProjectDir%\Tools" +set "__DotnetHost=%__ToolsDir%\dotnetcli\dotnet.exe" +if not exist "%__DotnetHost%" ( + echo %__DotnetHost% not found after init-tools. + exit /b 1 +) REM ========================================================================================= REM === @@ -212,7 +224,7 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -call "%__ProjectDir%\run.cmd" build -Project="%__NativeTestIntermediatesDir%\install.vcxproj" -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__msbuildNativeArgs% %__RunArgs% %__unprocessedBuildArgs% +call "%__ProjectDir%\run.cmd" build -Project="%__NativeTestIntermediatesDir%\install.vcxproj" -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__msbuildNativeArgs% %__RunArgs% %__PriorityArg% %__PassThroughArg% %__unprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% @@ -243,7 +255,7 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -BatchRestorePackages -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__unprocessedBuildArgs% +call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -BatchRestorePackages -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__PassThroughArg% %__unprocessedBuildArgs% set __BuildLogRootName=Tests_GenerateRuntimeLayout set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log @@ -253,7 +265,7 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\runtest.proj -BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct -RuntimeId="%__RuntimeId%" -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__unprocessedBuildArgs% +call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\runtest.proj -BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct -RuntimeId="%__RuntimeId%" -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__PassThroughArg% %__unprocessedBuildArgs% if errorlevel 1 ( echo BinPlace of mscorlib.dll failed exit /b 1 @@ -301,7 +313,7 @@ for /l %%G in (1, 1, %__BuildLoopCount%) do ( set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%";Append=!__AppendToLog! set TestBuildSlice=%%G - call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__unprocessedBuildArgs% + call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__PassThroughArg% %__unprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: @@ -331,7 +343,7 @@ md "%CORE_ROOT_STAGE%" xcopy "%__BinDir%" "%CORE_ROOT_STAGE%" -if defined __BuildAgainstPackagesArg ( +if defined __BuildAgainstPackagesArg ( if "%__TargetsWindows%"=="0" ( if not exist %__PackagesDir%\TestNativeBins ( @@ -347,16 +359,18 @@ if defined __BuildAgainstPackagesArg ( echo %__MsgPrefix%Creating test wrappers... set RuntimeIdArg= -set TargetsWindowsArg= - if defined __RuntimeId ( set RuntimeIdArg=-RuntimeID="%__RuntimeId%" ) +set TargetsWindowsArg= +set TargetsWindowsMsbuildArg= if "%__TargetsWindows%"=="1" ( set TargetsWindowsArg=-TargetsWindows=true + set TargetsWindowsMsbuildArg=/p:TargetsWindows=true ) else if "%__TargetsWindows%"=="0" ( set TargetsWindowsArg=-TargetsWindows=false + set TargetsWindowsMsbuildArg=/p:TargetsWindows=false ) set __BuildLogRootName=Tests_XunitWrapper @@ -367,11 +381,13 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -BuildWrappers -MsBuildEventLogging=" " -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %TargetsWindowsArg% %__unprocessedBuildArgs% +REM Build wrappers using the local SDK's msbuild. As we move to arcade, the other builds should be moved away from run.exe as well. +call %__DotnetHost% msbuild %__ProjectDir%\tests\runtest.proj /p:BuildWrappers=true !__msbuildLog! !__msbuildWrn! !__msbuildErr! %__msbuildArgs% %TargetsWindowsMsbuildArg% %__BuildAgainstPackagesMsbuildArg% %__unprocessedBuildArgs% if errorlevel 1 ( echo Xunit Wrapper build failed exit /b 1 ) +echo { "build_os": "%__BuildOS%", "build_arch": "%__BuildArch%", "build_type": "%__BuildType%" } > "%__TestBinDir%/build_info.json" echo %__MsgPrefix%Creating test overlay... @@ -383,7 +399,7 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testOverlay -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %RuntimeIdArg% %__unprocessedBuildArgs% +call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testOverlay -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %RuntimeIdArg% %__PriorityArg% %__PassThroughArg% %__unprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% @@ -422,7 +438,7 @@ REM === Prep test binaries for Helix publishing REM === REM ========================================================================================= -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\helixprep.proj -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %RuntimeIdArg% %TargetsWindowsArg% %__CrossgenArg% %__unprocessedBuildArgs% +call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\helixprep.proj -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %RuntimeIdArg% %TargetsWindowsArg% %__CrossgenArg% %__PriorityArg% %__PassThroughArg% %__unprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% @@ -550,6 +566,6 @@ if %__exitCode% neq 0 ( :: Delete original .dll & replace it with the Crossgened .dll del %1 ren "%CORE_ROOT%\temp.ni.dll" %2 - + echo Successfully precompiled %2 exit /b 0 diff --git a/buildpipeline/tests/Dotnet-CoreClr-Trusted-BuildTests.json b/buildpipeline/tests/Dotnet-CoreClr-Trusted-BuildTests.json index 3c65d64a304b..9f8c64980cfe 100644 --- a/buildpipeline/tests/Dotnet-CoreClr-Trusted-BuildTests.json +++ b/buildpipeline/tests/Dotnet-CoreClr-Trusted-BuildTests.json @@ -125,7 +125,7 @@ }, "inputs": { "filename": "build-test.cmd", - "arguments": "$(ParentBuildType) $(Architecture) buildagainstpackages runtimeid $(Rid) $(TargetsNonWindowsArg)$(CrossgenArg)-OfficialBuildId=$(ParentOfficialBuildId) -OverwriteCoreClrPackageVersion -Priority=$(Priority) -- /p:IntermediateAzureFeed=$(IntermediateAzureFeed)", + "arguments": "$(ParentBuildType) $(Architecture) buildagainstpackages runtimeid $(Rid) $(TargetsNonWindowsArg) $(CrossgenArg) -Priority=$(Priority) -- /p:OfficialBuildId=$(ParentOfficialBuildId) /p:OverwriteCoreClrPackageVersion=true /p:IntermediateAzureFeed=$(IntermediateAzureFeed)", "workingFolder": "", "failOnStandardError": "false" } diff --git a/dependencies.props b/dependencies.props index f60505e64ccc..aae838d8f03d 100644 --- a/dependencies.props +++ b/dependencies.props @@ -35,12 +35,11 @@ 2.2.0-preview3-27014-02 2.2.0-release-20180912-0041 2.2.0-preview3-27014-03 - 2.2.0-beta2-build3300 - 1.0.2-prerelease-00177 + 2.4.1-pre.build.4059 1.0.0-beta-build0015 2.0.4 2.1.1 - 1.2.0 + 2.0.0 + + + $(MSBuildThisFileDirectory) + $(CoreclrDir)/packages + + + + + $(__BuildArch) + x64 + x64 + + $(__BuildType) + Debug + Debug + Release + Checked + + $(__BuildOS) + Windows_NT + + $(BuildType) + $(BuildArch) + + + + $(PackagesDir) + + + + diff --git a/dir.props b/dir.props index 457216f14548..3a23ea595579 100644 --- a/dir.props +++ b/dir.props @@ -2,6 +2,8 @@ + + @@ -42,18 +44,6 @@ - - $(__BuildArch) - x64 - x64 - - Debug - Debug - Release - Checked - - $(__BuildOS) - Windows_NT $(__ProjectDir)\ $(MSBuildThisFileDirectory) diff --git a/init-tools.cmd b/init-tools.cmd index ec21fe808dac..7b8ab22021ef 100644 --- a/init-tools.cmd +++ b/init-tools.cmd @@ -47,9 +47,33 @@ echo Running %0 > "%INIT_TOOLS_LOG%" set /p DOTNET_VERSION=< "%~dp0DotnetCLIVersion.txt" if exist "%DOTNET_CMD%" goto :afterdotnetrestore +REM Use x86 tools on arm64 and x86. +REM arm32 host is not currently supported, please crossbuild. +if /i "%PROCESSOR_ARCHITECTURE%" == "arm" ( + echo "Error, arm32 arch not supported for build tools." + exit /b 1 +) + +if /i "%PROCESSOR_ARCHITECTURE%" == "amd64" ( + set _Arch=x64 + goto ArchSet +) + +REM If this is not amd64 and not arm, then we should be running on arm64 or x86 +REM either way we can (and should) use the x86 dotnet cli +REM +REM TODO: consume native arm64 toolset, blocked by official arm64 windows cli +REM : release. See https://github.com/dotnet/coreclr/issues/19614 for more +REM : information +set _Arch=x86 + +echo "init-tools.cmd: Setting arch to %_Arch% for build tools" + +:ArchSet + echo Installing dotnet cli... if NOT exist "%DOTNET_PATH%" mkdir "%DOTNET_PATH%" -set DOTNET_ZIP_NAME=dotnet-sdk-%DOTNET_VERSION%-win-x64.zip +set DOTNET_ZIP_NAME=dotnet-sdk-%DOTNET_VERSION%-win-%_Arch%.zip set DOTNET_REMOTE_PATH=https://dotnetcli.azureedge.net/dotnet/Sdk/%DOTNET_VERSION%/%DOTNET_ZIP_NAME% set DOTNET_LOCAL_PATH=%DOTNET_PATH%%DOTNET_ZIP_NAME% echo Installing '%DOTNET_REMOTE_PATH%' to '%DOTNET_LOCAL_PATH%' >> "%INIT_TOOLS_LOG%" @@ -61,6 +85,15 @@ if NOT exist "%DOTNET_LOCAL_PATH%" ( :afterdotnetrestore +REM We do not need the build tools for arm64/x86 +if /i "%PROCESSOR_ARCHITEW6432%" == "arm64" ( + goto :EOF +) + +if /i "%PROCESSOR_ARCHITECTURE%" == "arm64" ( + goto :EOF +) + if exist "%BUILD_TOOLS_PATH%" goto :afterbuildtoolsrestore echo Restoring BuildTools version %BUILDTOOLS_VERSION%... echo Running: "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages %PACKAGES_DIR% --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" diff --git a/netci.groovy b/netci.groovy index efc21e2027bd..8a46fc438b9f 100755 --- a/netci.groovy +++ b/netci.groovy @@ -228,10 +228,10 @@ class Constants { // Note that some scenarios that are commented out should be enabled, but haven't yet been. // def static validArmWindowsScenarios = [ - 'innerloop': [], - 'normal': [], + 'innerloop', + 'normal', // 'ilrt' - 'r2r': ["R2R_FAIL", "R2R_EXCLUDE"], + 'r2r', // 'longgc' // 'formatting' // 'gcsimulator' @@ -239,102 +239,102 @@ class Constants { // 'standalone_gc' // 'gc_reliability_framework' // 'illink' - 'r2r_jitstress1': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstress2': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstress1_tiered': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstress2_tiered': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs1': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs2': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs3': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs4': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs8': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs0x10': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs0x80': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitstressregs0x1000': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_jitminopts': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"], - 'r2r_jitforcerelocs': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'r2r_gcstress15': ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE", "GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"], - 'r2r_no_tiered_compilation': ["R2R_FAIL", "R2R_EXCLUDE"], - 'minopts': ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"], - 'tieredcompilation': [], - 'no_tiered_compilation': [], - 'no_tiered_compilation_innerloop': [], - 'forcerelocs': [], - 'jitstress1': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress1_tiered': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_tiered': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs1': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs2': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs3': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs4': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs8': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs0x10': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs0x80': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstressregs0x1000': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs1': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs2': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs3': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs4': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs8': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs0x10': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs0x80': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'jitstress2_jitstressregs0x1000': ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'tailcallstress': ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"], - // 'jitsse2only' // Only relevant to xarch - 'jitnosimd': [], // Only interesting on platforms where SIMD support exists. + 'r2r_jitstress1', + 'r2r_jitstress2', + 'r2r_jitstress1_tiered', + 'r2r_jitstress2_tiered', + 'r2r_jitstressregs1', + 'r2r_jitstressregs2', + 'r2r_jitstressregs3', + 'r2r_jitstressregs4', + 'r2r_jitstressregs8', + 'r2r_jitstressregs0x10', + 'r2r_jitstressregs0x80', + 'r2r_jitstressregs0x1000', + 'r2r_jitminopts', + 'r2r_jitforcerelocs', + 'r2r_gcstress15', + 'r2r_no_tiered_compilation', + 'minopts', + 'tieredcompilation', + 'no_tiered_compilation', + 'no_tiered_compilation_innerloop', + 'forcerelocs', + 'jitstress1', + 'jitstress2', + 'jitstress1_tiered', + 'jitstress2_tiered', + 'jitstressregs1', + 'jitstressregs2', + 'jitstressregs3', + 'jitstressregs4', + 'jitstressregs8', + 'jitstressregs0x10', + 'jitstressregs0x80', + 'jitstressregs0x1000', + 'jitstress2_jitstressregs1', + 'jitstress2_jitstressregs2', + 'jitstress2_jitstressregs3', + 'jitstress2_jitstressregs4', + 'jitstress2_jitstressregs8', + 'jitstress2_jitstressregs0x10', + 'jitstress2_jitstressregs0x80', + 'jitstress2_jitstressregs0x1000', + 'tailcallstress', + // 'jitsse2only' // Only relevant to xarch + 'jitnosimd', // Only interesting on platforms where SIMD support exists. // 'jitincompletehwintrinsic' // 'jitx86hwintrinsicnoavx' // 'jitx86hwintrinsicnoavx2' // 'jitx86hwintrinsicnosimd' // 'jitnox86hwintrinsic' - 'corefx_baseline': [], // corefx tests don't use smarty - 'corefx_minopts': [], // corefx tests don't use smarty - 'corefx_tieredcompilation': [], // corefx tests don't use smarty - 'corefx_jitstress1': [], // corefx tests don't use smarty - 'corefx_jitstress2': [], // corefx tests don't use smarty - 'corefx_jitstressregs1': [], // corefx tests don't use smarty - 'corefx_jitstressregs2': [], // corefx tests don't use smarty - 'corefx_jitstressregs3': [], // corefx tests don't use smarty - 'corefx_jitstressregs4': [], // corefx tests don't use smarty - 'corefx_jitstressregs8': [], // corefx tests don't use smarty - 'corefx_jitstressregs0x10': [], // corefx tests don't use smarty - 'corefx_jitstressregs0x80': [], // corefx tests don't use smarty - 'corefx_jitstressregs0x1000': [], // corefx tests don't use smarty - 'gcstress0x3': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"], - 'gcstress0xc': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"], - 'zapdisable': ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"], - 'heapverify1': [], - 'gcstress0xc_zapdisable': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"], - 'gcstress0xc_zapdisable_jitstress2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_zapdisable_heapverify1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"], - 'gcstress0xc_jitstress1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstress2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_minopts_heapverify1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"], + 'corefx_baseline', + 'corefx_minopts', + 'corefx_tieredcompilation', + 'corefx_jitstress1', + 'corefx_jitstress2', + 'corefx_jitstressregs1', + 'corefx_jitstressregs2', + 'corefx_jitstressregs3', + 'corefx_jitstressregs4', + 'corefx_jitstressregs8', + 'corefx_jitstressregs0x10', + 'corefx_jitstressregs0x80', + 'corefx_jitstressregs0x1000', + 'gcstress0x3', + 'gcstress0xc', + 'zapdisable', + 'heapverify1', + 'gcstress0xc_zapdisable', + 'gcstress0xc_zapdisable_jitstress2', + 'gcstress0xc_zapdisable_heapverify1', + 'gcstress0xc_jitstress1', + 'gcstress0xc_jitstress2', + 'gcstress0xc_minopts_heapverify1', // // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug? // - 'minopts_zapdisable': ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"], - 'gcstress0x3_jitstress1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstress2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs3': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs4': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs8': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs0x10': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs0x80': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0x3_jitstressregs0x1000': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs1': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs2': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs3': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs4': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs8': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs0x10': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs0x80': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"], - 'gcstress0xc_jitstressregs0x1000': ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"] + 'minopts_zapdisable', + 'gcstress0x3_jitstress1', + 'gcstress0x3_jitstress2', + 'gcstress0x3_jitstressregs1', + 'gcstress0x3_jitstressregs2', + 'gcstress0x3_jitstressregs3', + 'gcstress0x3_jitstressregs4', + 'gcstress0x3_jitstressregs8', + 'gcstress0x3_jitstressregs0x10', + 'gcstress0x3_jitstressregs0x80', + 'gcstress0x3_jitstressregs0x1000', + 'gcstress0xc_jitstressregs1', + 'gcstress0xc_jitstressregs2', + 'gcstress0xc_jitstressregs3', + 'gcstress0xc_jitstressregs4', + 'gcstress0xc_jitstressregs8', + 'gcstress0xc_jitstressregs0x10', + 'gcstress0xc_jitstressregs0x80', + 'gcstress0xc_jitstressregs0x1000' ] def static validLinuxArm64Scenarios = [ @@ -605,7 +605,7 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n if (isBuild == true) { Utilities.setMachineAffinity(job, os, 'latest-arm64') } else { - Utilities.setMachineAffinity(job, os, 'arm64-windows_nt') + Utilities.setMachineAffinity(job, 'windows.10.arm64.open') } } else { assert os != 'Windows_NT' @@ -751,7 +751,7 @@ def static isGcReliabilityFramework(def scenario) { } def static isArmWindowsScenario(def scenario) { - return Constants.validArmWindowsScenarios.containsKey(scenario) + return Constants.validArmWindowsScenarios.contains(scenario) } def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) { @@ -2961,60 +2961,15 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture addEnvVariable("RunCrossGen", "true") } // isR2RScenario(scenario) - // Create the smarty command - def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS " - def addSmartyFlag = { flag -> smartyCommand += flag + " "} - def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)} - def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } } - - if (architecture == 'armlb') { - addExclude("LEGACYJIT_FAIL") - } - - // Exclude tests based on scenario. - Constants.validArmWindowsScenarios[scenario].each { excludeTag -> - addArchSpecificExclude(architecture, excludeTag) - } - - if (isPri0TestScenario(scenario)) { - addExclude("pri1") - } + // Run runtest.cmd + // Do not run generate layout. It will delete the correct CORE_ROOT, and we do not have a correct product + // dir to copy from. + def runtestCommand = "%WORKSPACE%\\tests\\runtest.cmd ${architecture} ${configuration} skipgeneratelayout" - // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result. - // TODO: We should create a "long running" job that runs these with a longer timeout. - addExclude("LONG_RUNNING") - - smartyCommand += "/lstFile Tests.lst" - - def testListArch = [ - 'arm64': 'arm64', - 'arm': 'arm', - 'armlb': 'arm' - ] - - def archLocation = testListArch[architecture] - - addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}") - addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}") - addCommand("${smartyCommand}") - - // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file. - // However, we also need to remove all the variables that were set during this batch file, so we - // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell - // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well - // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel% - // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command. - // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity. - addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%") - - // ZIP up the smarty output, no matter what the smarty result. - addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"") - - addCommand("echo %errorlevel%") - addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}") + addCommand("${runtestCommand}") // Use the smarty errorlevel as the script errorlevel. - addCommand("exit /b %__save_smarty_errorlevel%") + addCommand("exit /b %errorlevel%") batchFile(buildCommands) } // non-corefx testing @@ -3022,12 +2977,7 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture } // job if (!isCoreFxScenario(scenario)) { - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false) - - // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much, - // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get - // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large. - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false) + Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true) } return newJob diff --git a/run.sh b/run.sh index 2d037e91e557..cc6a1846bbeb 100755 --- a/run.sh +++ b/run.sh @@ -9,7 +9,7 @@ toolRuntime=$working_tree_root/Tools dotnet=$toolRuntime/dotnetcli/dotnet echo "Running: $dotnet $toolRuntime/run.exe $working_tree_root/config.json $*" -$dotnet $toolRuntime/run.exe $working_tree_root/config.json $* +$dotnet $toolRuntime/run.exe $working_tree_root/config.json "$@" if [ $? -ne 0 ] then echo "ERROR: An error occured in $dotnet $toolRuntime/run $#. Check $# logs under $working_tree_root." diff --git a/tests/build.proj b/tests/build.proj index aede1317a1c3..1235c17cc7b8 100644 --- a/tests/build.proj +++ b/tests/build.proj @@ -26,12 +26,11 @@ - + - diff --git a/tests/dir.common.props b/tests/dir.common.props new file mode 100644 index 000000000000..5c596c0fed86 --- /dev/null +++ b/tests/dir.common.props @@ -0,0 +1,28 @@ + + + + + + + $(BuildOS).$(Platform).$(Configuration) + + $(CoreclrDir)/tests/src + $([System.String]::Copy('$(MSBuildProjectDirectary)').Replace($(TestSrcDir),''))/$(MSBuildProjectName) + + + $(CoreclrDir)/bin/tests/obj/$(OSPlatformConfig)/Managed/$(BuildProjectRelativeDir) + $(BaseIntermediateOutputPath) + + $(CoreclrDir)/bin/tests/$(OSPlatformConfig)/$(BuildProjectRelativeDir) + $(BaseOutputPath) + + + + diff --git a/tests/dir.props b/tests/dir.props index b29983cfbe9e..5b0713610d98 100644 --- a/tests/dir.props +++ b/tests/dir.props @@ -34,7 +34,6 @@ $(BuildToolsTargetsDesktop) true true - false @@ -47,7 +46,6 @@ $(ToolsDir)dotnetcli\ $(ToolsDir)net46\ $(DotnetCliPath)dotnet - $(ToolsDir)\Microsoft.CSharp.Core.targets <_TargetFrameworkDirectories Condition="'$(BuildToolsTargetsDesktop)' != 'true'">$(MSBuildThisFileDirectory)/Documentation @@ -105,8 +103,7 @@ - - + diff --git a/tests/dir.sdkbuild.props b/tests/dir.sdkbuild.props new file mode 100644 index 000000000000..b5a0fdd07662 --- /dev/null +++ b/tests/dir.sdkbuild.props @@ -0,0 +1,14 @@ + + + + + + + netcoreapp2.0 + false + false + + + diff --git a/tests/helixprep.proj b/tests/helixprep.proj index 2d08e91b13c6..39fa2a60627a 100644 --- a/tests/helixprep.proj +++ b/tests/helixprep.proj @@ -1,7 +1,7 @@ - + @@ -37,8 +37,8 @@ - - @@ -85,8 +85,8 @@ set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD% $(CmdCrossgenVar) ECHO BEGIN EXECUTION -ECHO %HELIX_CORRELATION_PAYLOAD%\CoreRun.exe %HELIX_WORKITEM_PAYLOAD%\xunit.console.netcore.exe %HELIX_WORKITEM_PAYLOAD%\$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing -%HELIX_CORRELATION_PAYLOAD%\CoreRun.exe %HELIX_WORKITEM_PAYLOAD%\xunit.console.netcore.exe %HELIX_WORKITEM_PAYLOAD%\$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing +ECHO %HELIX_CORRELATION_PAYLOAD%\CoreRun.exe %HELIX_WORKITEM_PAYLOAD%\xunit.console.dll %HELIX_WORKITEM_PAYLOAD%\$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing +%HELIX_CORRELATION_PAYLOAD%\CoreRun.exe %HELIX_WORKITEM_PAYLOAD%\xunit.console.dll %HELIX_WORKITEM_PAYLOAD%\$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing echo Finished running tests. Exit code = %ERRORLEVEL% EXIT /B %ERRORLEVEL% @@ -109,7 +109,7 @@ EXIT /B %ERRORLEVEL% #!/bin/sh%0a - $(WrapperShContents)%0a + $(WrapperShContents)%0a $(WrapperShContents)export CORE_ROOT="%24HELIX_CORRELATION_PAYLOAD"%0a $(WrapperShContents)%0a $(WrapperShContents)for scriptFilePath in %24(find . -type f -iname '%2A.sh' ! -iname "runtests.sh" | sort)%0a @@ -119,7 +119,7 @@ EXIT /B %ERRORLEVEL% $(WrapperShContents)done%0a $(WrapperShContents)%0a - $(WrapperShContents)export RunCrossGen=true%0a $(WrapperShContents)overlayDir=%24CORE_ROOT%0a @@ -140,7 +140,7 @@ EXIT /B %ERRORLEVEL% $(WrapperShContents) else%0a $(WrapperShContents) rm %24filename%0a $(WrapperShContents) mv %24overlayDir/temp.ni.dll %24filename%0a - $(WrapperShContents) fi%0a + $(WrapperShContents) fi%0a $(WrapperShContents) rm %24filename.stdout%0a $(WrapperShContents) rm %24filename.stderr%0a $(WrapperShContents) done%0a @@ -149,8 +149,8 @@ EXIT /B %ERRORLEVEL% $(WrapperShContents)%0a $(WrapperShContents)echo BEGIN EXECUTION%0a - $(WrapperShContents)echo "%24{HELIX_CORRELATION_PAYLOAD}/corerun" %24HELIX_WORKITEM_PAYLOAD/xunit.console.netcore.exe %24HELIX_WORKITEM_PAYLOAD/$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing%0a - $(WrapperShContents)"%24{HELIX_CORRELATION_PAYLOAD}/corerun" %24HELIX_WORKITEM_PAYLOAD/xunit.console.netcore.exe %24HELIX_WORKITEM_PAYLOAD/$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing%0a + $(WrapperShContents)echo "%24{HELIX_CORRELATION_PAYLOAD}/corerun" %24HELIX_WORKITEM_PAYLOAD/xunit.console.dll %24HELIX_WORKITEM_PAYLOAD/$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing%0a + $(WrapperShContents)"%24{HELIX_CORRELATION_PAYLOAD}/corerun" %24HELIX_WORKITEM_PAYLOAD/xunit.console.dll %24HELIX_WORKITEM_PAYLOAD/$(ProjectName) -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing%0a $(WrapperShContents)ErrorLevel=%24%3F%0a $(WrapperShContents)%0a $(WrapperShContents)echo Finished running tests. Exit code = %24ErrorLevel%0a @@ -184,4 +184,4 @@ EXIT /B %ERRORLEVEL% Targets="GenerateWrapperExecutables;ArchiveTests;ArchiveCoreRoot" /> - \ No newline at end of file + diff --git a/tests/issues.targets b/tests/issues.targets index ef8dd508a57a..e4afb18f92eb 100644 --- a/tests/issues.targets +++ b/tests/issues.targets @@ -255,18 +255,188 @@ 7163, fails on both legacy backend and RyuJIT - 11469, The test causes OutOfMemory exception in crossgen mode. + 11469, The test causes OutOfMemory exception in crossgen mode. + - - 6217 + + needs triage - - 2420 + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + 19218 + + + 19218 + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + + + + times out + + + needs triage + + + needs triage + + + needs triage + + + times out + + + times out + + + needs triage + + + times out + + + times out + + + needs triage + + + times out + + + times out + + + needs triage + + + times out + + + times out + + + needs triage + + + times out + + + times out + + + times out + + + times out + + + 19537 + + + 19537 + + + + + + + 8648 + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + 18895 + + + 18989 + + + needs triage + + + needs triage + + + needs triage + + + 19441 @@ -1761,7 +1931,7 @@ - + - - - - - - + $(XUnitTestBinBase)\$(CategoryWithSlash) + + + + - - - - - {8ffe99c0-22f8-4462-b839-970eac1b3472} - coreclr - - - {8ffe99c0-22f8-4462-b839-970eac1b3472} - coreclr - + - - - + + - - - $(XunitTestBinBase)\$(CategoryWithSlash)\ - %24(TestWrappersPackagesConfigFileDirectory)obj/project.assets.json - + + + - ]]> +]]> - @@ -143,7 +113,7 @@ $(_XunitEpilog) - + @@ -158,7 +128,6 @@ $(_XunitEpilog) $([System.String]::Copy('$(CategoryWithSlash)').Replace('/','.')) $(Category).XUnitWrapper $(XunitWrapperGeneratedCSDirBase)$(Category) - $(XunitWrapperOutputIntermediatedDirBase)$(Category) <_XunitProlog Condition=" '$(_XunitProlog)'=='' "> @@ -183,7 +152,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). { reportBase = System.Environment.GetEnvironmentVariable(%22XunitTestReportDirBase%22)%3B testBinaryBase = System.IO.Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).LocalPath)%3B - coreRoot = System.IO.Path.GetFullPath(System.Environment.GetEnvironmentVariable(%22CORE_ROOT%22))%3B + coreRoot = System.Environment.GetEnvironmentVariable(%22CORE_ROOT%22)%3B if (String.IsNullOrEmpty(reportBase)) { reportBase = System.IO.Path.Combine(testBinaryBase, "Reports")%3B @@ -197,6 +166,8 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). throw new ArgumentException("Environment variable CORE_ROOT is not set")%3B } + coreRoot = System.IO.Path.GetFullPath(coreRoot)%3B + string operatingSystem = System.Environment.GetEnvironmentVariable("OS")%3B runningInWindows = (operatingSystem != null && operatingSystem.StartsWith("Windows"))%3B } @@ -250,7 +221,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). try { CoreclrTestWrapperLib wrapper = new CoreclrTestWrapperLib()%3B - string testSubfolder = @"\$(Category)\$([System.String]::Copy('%(AllCMDs.RelativeDir)').Replace("$(_CMDDIR)\",''))"%3B + string testSubfolder = @"\$(Category)\$([System.String]::Copy('%(AllCMDs.RelativeDir)').Replace("$(_CMDDIR)$([System.IO.Path]::DirectorySeparatorChar)",''))"%3B outputFile = System.IO.Path.GetFullPath(_Global.reportBase + testSubfolder + @"%(AllCMDs.FileName).output.txt")%3B errorFile = System.IO.Path.GetFullPath(_Global.reportBase + testSubfolder + @"%(AllCMDs.FileName).error.txt")%3B testExecutable = System.IO.Path.GetFullPath(_Global.testBinaryBase + @"$([System.String]::Copy('%(AllCMDs.FullPath)').Replace("$(_CMDDIR)",''))")%3B @@ -278,7 +249,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). } catch(Exception ex) { - sErrorText = "Unable to read error file: " + errorFile%3B + sErrorText = $"Unable to read error file: {errorFile}\n{ex}"%3B } string outputText = null%3B @@ -290,10 +261,10 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). } catch(Exception ex) { - outputText = "Unable to read output file: " + outputFile%3B + outputText = $"Unable to read error file: {outputFile}\n{ex}"%3B } - string msg = infraEx != null ? "Test Infrastructure Failure: " + infraEx.Message + string msg = infraEx != null ? "Test Infrastructure Failure: " + infraEx.ToString() : sErrorText + "\n\n" + "Return code: " + ret + "\n" + "Raw output file: " + outputFile + "\n" + @@ -313,6 +284,9 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). + + + @@ -321,7 +295,8 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). - + + @@ -401,6 +376,6 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). - + diff --git a/tests/runtest.py b/tests/runtest.py new file mode 100755 index 000000000000..cd21c3dc5e14 --- /dev/null +++ b/tests/runtest.py @@ -0,0 +1,2034 @@ +#!/usr/bin/env python +# +## Licensed to the .NET Foundation under one or more agreements. +## The .NET Foundation licenses this file to you under the MIT license. +## See the LICENSE file in the project root for more information. +# +## +# Title :runtest.py +# +# Notes: +# +# Universal script to setup and run the xunit console runner. The script relies +# on runtest.proj and the bash and batch wrappers. All test excludes will also +# come from issues.targets. If there is a jit stress or gc stress exclude, +# please add GCStressIncompatible or JitOptimizationSensitive to the test's +# ilproj or csproj. +# +# The xunit runner currently relies on tests being built on the same host as the +# target platform. This requires all tests run on linux x64 to be built by the +# same platform and arch. If this is not done, the tests will run correctly; +# however, expect failures due to incorrect exclusions in the xunit +# wrappers setup at build time. +# +# Note that for linux targets the native components to the tests are still built +# by the product build. This requires all native components to be either copied +# into the Core_Root directory or the test's managed directory. The latter is +# prone to failure; however, copying into the Core_Root directory may create +# naming conflicts. +# +# If you are running tests on a different target than the host that built, the +# native tests components must be copied from: +# bin/obj/.... 0: + print("Found COMPlus variables in the current environment") + print("") + + file_header = None + + if _os == "Windows_NT": + file_header = \ +"""@echo off +REM Temporary test env for test run. + +""" + else: + file_header = \ +"""# Temporary test env for test run. + +""" + + contents = "" + + with tempfile.NamedTemporaryFile(mode="w") as test_env: + test_env.write(file_header) + contents += file_header + + for key in complus_vars: + value = complus_vars[key] + command = None + if _os == "Windows_NT": + command = "set" + else: + command = "export" + + print("Unset %s" % key) + if key.lower() == "complus_gcstress" and "c" in value.lower(): + gc_stress_c = True + + if key.lower() == "complus_gcstress": + gc_stress = True + + os.environ[key] = "" + + line = "%s %s=%s%s" % (command, key, value, os.linesep) + test_env.write(line) + contents += line + + print("") + print("TestEnv: %s" % test_env.name) + print("") + print("Contents:") + print("") + print(contents) + print("") + + return func(test_env.name) + + else: + return func(None) + +def get_environment(test_env=None): + """ Get all the COMPlus_* Environment variables + + Notes: + All COMPlus variables need to be captured as a test_env script to avoid + influencing the test runner. + """ + global gc_stress_c + + complus_vars = defaultdict(lambda: "") + + for key in os.environ: + if "complus" in key.lower(): + complus_vars[key] = os.environ[key] + os.environ[key] = '' + elif "superpmi" in key.lower(): + complus_vars[key] = os.environ[key] + os.environ[key] = '' + + # Get the env from the test_env + if test_env is not None: + with open(test_env) as file_handle: + for item in file_handle.readlines(): + key_split = item.split("=") + + if len(key_split) == 1: + continue + + key = key_split[0] + value = key_split[1] + + key = key.split(" ")[-1] + value = value.strip() + + try: + value = value.split(" ")[0] + except: + pass + + complus_vars[key] = value + + # Supoort looking up case insensitive. + complus_vars[key.lower()] = value + + if "complus_gcstress" in complus_vars: + gc_stress = True + + if "c" in complus_vars["COMPlus_GCStress"].lower(): + gc_stress_c = True + + return complus_vars + +def call_msbuild(coreclr_repo_location, + dotnetcli_location, + host_os, + arch, + build_type, + is_illink=False, + sequential=False): + """ Call msbuild to run the tests built. + + Args: + coreclr_repo_location(str) : path to coreclr repo + dotnetcli_location(str) : path to the dotnet cli in the tools dir + sequential(bool) : run sequentially if True + + host_os(str) : os + arch(str) : architecture + build_type(str) : configuration + + Notes: + At this point the environment should be setup correctly, including + the test_env, should it need to be passed. + + """ + global g_verbose + + common_msbuild_arguments = [] + + if sequential: + common_msbuild_arguments += ["/p:ParallelRun=none"] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + command = [dotnetcli_location, + "msbuild", + os.path.join(coreclr_repo_location, "tests", "runtest.proj"), + "/p:Runtests=true", + "/clp:showcommandline"] + + command += common_msbuild_arguments + + if is_illink: + command += ["/p:RunTestsViaIllink=true"] + + log_path = os.path.join(logs_dir, "TestRunResults_%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + msbuild_log_args = ["/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log, + "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log, + "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log, + "/consoleloggerparameters:Summary"] + + if g_verbose: + msbuild_log_args += ["/verbosity:diag"] + + command += msbuild_log_args + + command += ["/p:__BuildOS=%s" % host_os, + "/p:__BuildArch=%s" % arch, + "/p:__BuildType=%s" % build_type, + "/p:__LogsDir=%s" % logs_dir] + + print(" ".join(command)) + + sys.stdout.flush() # flush output before creating sub-process + proc = subprocess.Popen(command) + + try: + proc.communicate() + except: + proc.kill() + sys.exit(1) + + return proc.returncode + +def running_in_ci(): + """ Check if running in ci + + Returns: + bool + """ + + is_ci = False + + try: + jenkins_build_number = os.environ["BUILD_NUMBER"] + + is_ci = True + except: + pass + + return is_ci + +def copy_native_test_bin_to_core_root(host_os, path, core_root): + """ Recursively copy all files to core_root + + Args: + host_os(str) : os + path(str) : native test bin location + core_root(str) : core_root location + """ + assert os.path.isdir(path) or os.path.isfile(path) + assert os.path.isdir(core_root) + + extension = "so" if host_os == "Linux" else "dylib" + + if os.path.isdir(path): + for item in os.listdir(path): + copy_native_test_bin_to_core_root(host_os, os.path.join(path, item), core_root) + elif path.endswith(extension): + print("cp -p %s %s" % (path, core_root)) + shutil.copy2(path, core_root) + +def correct_line_endings(host_os, test_location, root=True): + """ Recursively correct all .sh/.cmd files to the correct line ending + + Args: + host_os(str) : os + test_location(str) : location of the tests + """ + if root: + print("Correcting line endings...") + + assert os.path.isdir(test_location) or os.path.isfile(test_location) + + extension = "cmd" if host_os == "Windows_NT" else ".sh" + incorrect_line_ending = '\n' if host_os == "Windows_NT" else '\r\n' + correct_line_ending = os.linesep + + if os.path.isdir(test_location): + for item in os.listdir(test_location): + correct_line_endings(host_os, os.path.join(test_location, item), False) + elif test_location.endswith(extension): + if sys.version_info < (3,0): + + content = None + with open(test_location) as file_handle: + content = file_handle.read() + + assert content != None + subbed_content = content.replace(incorrect_line_ending, correct_line_ending) + + if content != subbed_content: + with open(test_location, 'w') as file_handle: + file_handle.write(subbed_content) + + else: + # Python3 will correct line endings automatically. + + content = None + with open(test_location) as file_handle: + content = file_handle.read() + + with open(test_location, 'w') as file_handle: + file_handle.write(content) + +def run_tests(host_os, + arch, + build_type, + core_root, + coreclr_repo_location, + test_location, + test_native_bin_location, + test_env=None, + is_long_gc=False, + is_gcsimulator=False, + is_jitdasm=False, + is_ilasm=False, + is_illink=False, + run_crossgen_tests=False, + run_sequential=False): + """ Run the coreclr tests + + Args: + host_os(str) : os + arch(str) : arch + build_type(str) : configuration + coreclr_repo_location(str) : path to the root of the repo + core_root(str) : Core_Root path + test_location(str) : Test bin, location + test_native_bin_location : Native test components, None and windows. + test_env(str) : path to the test_env to be used + """ + global gc_stress + + # Setup the dotnetcli location + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + # Default timeout for unix is 15 minutes + os.environ["__TestTimeout"] = str(15*60*1000) # 900,000 ms + + # Setup the environment + if is_long_gc: + print("Running Long GC Tests, extending timeout to 20 minutes.") + os.environ["__TestTimeout"] = str(20*60*1000) # 1,200,000 ms + os.environ["RunningLongGCTests"] = "1" + + if is_gcsimulator: + print("Running GCSimulator tests, extending timeout to one hour.") + os.environ["__TestTimeout"] = str(60*60*1000) # 3,600,000 ms + os.environ["RunningGCSimulatorTests"] = "1" + + if is_jitdasm: + print("Running jit disasm and tests.") + os.environ["RunningJitDisasm"] = "1" + + if is_ilasm: + print("Running ILasm round trip.") + os.environ["RunningIlasmRoundTrip"] = "1" + + if run_crossgen_tests: + print("Running tests R2R") + os.environ["RunCrossGen"] = "true" + + if gc_stress: + print("Running GCStress, extending timeout to 120 minutes.") + os.environ["__TestTimeout"] = str(120*60*1000) # 1,800,000 ms + + # Set Core_Root + print("Setting CORE_ROOT=%s" % core_root) + os.environ["CORE_ROOT"] = core_root + + # Set test env if exists + if test_env is not None: + os.environ["__TestEnv"] = test_env + + # Call msbuild. + return call_msbuild(coreclr_repo_location, + dotnetcli_location, + host_os, + arch, + build_type, + is_illink=is_illink, + sequential=run_sequential) + +def setup_args(args): + """ Setup the args based on the argparser obj + + Args: + args(ArgParser): Parsed arguments + + Notes: + If there is no core_root, or test location passed, create a default + location using the build type and the arch. + """ + + if args.generate_layout_only: + args.generate_layout = True + + host_os = None + arch = args.arch.lower() + build_type = args.build_type + + test_location = args.test_location + core_root = args.core_root + test_native_bin_location = args.test_native_bin_location + + coreclr_repo_location = args.coreclr_repo_location + if os.path.basename(coreclr_repo_location) == "tests": + coreclr_repo_location = os.path.dirname(coreclr_repo_location) + + if _platform == "linux" or _platform == "linux2": + host_os = "Linux" + elif _platform == "darwin": + host_os = "OSX" + elif _platform == "win32": + host_os = "Windows_NT" + else: + print("Unknown OS: %s" % host_os) + sys.exit(1) + + assert os.path.isdir(coreclr_repo_location) + + valid_arches = ["x64", "x86", "arm", "arm64"] + if not arch in valid_arches: + print("Unsupported architecture: %s." % arch) + print("Supported architectures: %s" % "[%s]" % ", ".join(valid_arches)) + sys.exit(1) + + def check_build_type(build_type): + valid_build_types = ["Debug", "Checked", "Release"] + + if build_type != None and len(build_type) > 0: + # Force the build type to be capitalized + build_type = build_type.capitalize() + + if not build_type in valid_build_types: + print("Unsupported configuration: %s." % build_type) + print("Supported configurations: %s" % "[%s]" % ", ".join(valid_build_types)) + sys.exit(1) + + return build_type + + build_type = check_build_type(build_type) + + if test_location is None: + default_test_location = os.path.join(coreclr_repo_location, "bin", "tests", "%s.%s.%s" % (host_os, arch, build_type)) + + if os.path.isdir(default_test_location): + test_location = default_test_location + + print("Using default test location.") + print("TestLocation: %s" % default_test_location) + print("") + + else: + # The tests for the default location have not been built. + print("Error, unable to find the tests at %s" % default_test_location) + + suggested_location = None + possible_test_locations = [item for item in os.listdir(os.path.join(coreclr_repo_location, "bin", "tests")) if host_os in item and arch in item] + if len(possible_test_locations) > 0: + print("Tests are built for the following:") + for item in possible_test_locations: + print(item.replace(".", " ")) + + print("Please run runtest.py again with the correct build-type by passing -build_type") + else: + print("No tests have been built for this host and arch. Please run build-test.%s" % ("cmd" if host_os == "Windows_NT" else "sh")) + + sys.exit(1) + else: + # If we have supplied our own test location then we need to create a test location + # that the scripting will expect. As it is now, there is a dependency on the + # test location being under test/.. + + # Make sure that we are using the correct build_type. This is a test drop, it is possible + # that we are inferring the build type to be Debug incorrectly. + + if build_type not in test_location: + # Remove punctuation + corrected_build_type = re.sub("[%s]" % string.punctuation, "", test_location.split(".")[-1]) + build_type = check_build_type(corrected_build_type) + + default_test_location = os.path.join(coreclr_repo_location, "bin", "tests", "%s.%s.%s" % (host_os, arch, build_type)) + + # Remove optional end os.path.sep + if test_location[-1] == os.path.sep: + test_location = test_location[:-1] + + if test_location.lower() != default_test_location.lower() and os.path.isdir(default_test_location): + # Remove the existing directory if there is one. + shutil.rmtree(default_test_location) + + print("Non-standard test location being used.") + print("Overwrite the standard location with these tests.") + print("TODO: Change runtest.proj to allow running from non-standard test location.") + print("") + + print("cp -r %s %s" % (test_location, default_test_location)) + shutil.copytree(test_location, default_test_location) + + test_location = default_test_location + + # unset core_root so it can be put in the default location + core_root = None + + # Force the core_root to be setup again. + args.generate_layout = True + + else: + test_location = default_test_location + + print("Using default test location.") + print("TestLocation: %s" % default_test_location) + print("") + + if core_root is None: + default_core_root = os.path.join(test_location, "Tests", "Core_Root") + + if os.path.isdir(default_core_root): + core_root = default_core_root + + print("Using default location for core_root.") + print("Core_Root: %s" % core_root) + print("") + + elif args.generate_layout is False: + # CORE_ROOT has not been setup correctly. + print("Error, unable to find CORE_ROOT at %s" % default_core_root) + print("Please run runtest.py with --generate_layout specified.") + + sys.exit(1) + + else: + print("--generate_layout passed. Core_Root will be populated at: %s" % default_core_root) + core_root = default_core_root + else: + print("Core_Root: %s" % core_root) + + is_same_os = False + is_same_arch = False + is_same_build_type = False + + # We will write out build information into the test directory. This is used + # by runtest.py to determine whether we need to rebuild the test wrappers. + if os.path.isfile(os.path.join(test_location, "build_info.json")): + with open(os.path.join(test_location, "build_info.json")) as file_handle: + build_info = json.load(file_handle) + is_same_os = build_info["build_os"] == host_os + is_same_arch = build_info["build_arch"] == arch + is_same_build_type = build_info["build_type"] == build_type + + if host_os != "Windows_NT" and not (is_same_os and is_same_arch and is_same_build_type): + if test_native_bin_location is None: + print("Using default location for test_native_bin_location.") + test_native_bin_location = os.path.join(os.path.join(coreclr_repo_location, "bin", "obj", "%s.%s.%s" % (host_os, arch, build_type), "tests")) + print("Native bin location: %s" % test_native_bin_location) + print("") + + if not os.path.isdir(test_native_bin_location): + print("Error, test_native_bin_location: %s, does not exist." % test_native_bin_location) + sys.exit(1) + + if args.product_location is None and args.generate_layout: + product_location = os.path.join(coreclr_repo_location, "bin", "Product", "%s.%s.%s" % (host_os, arch, build_type)) + if not os.path.isdir(product_location): + print("Error, unable to determine the product location. This is most likely because build_type was") + print("incorrectly passed. Or the product is not built. Please explicitely pass -product_location") + + sys.exit(1) + + else: + product_location = args.product_location + + return host_os, arch, build_type, coreclr_repo_location, product_location, core_root, test_location, test_native_bin_location + +def setup_tools(host_os, coreclr_repo_location): + """ Setup the tools for the repo + + Args: + host_os(str) : os + coreclr_repo_location(str) : path to coreclr repo + + """ + + # Is the tools dir setup + setup = False + tools_dir = os.path.join(coreclr_repo_location, "Tools") + + is_windows = host_os == "Windows_NT" + + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + if os.path.isfile(dotnetcli_location): + setup = True + + # init the tools for the repo + if not setup: + command = None + if is_windows: + command = [os.path.join(coreclr_repo_location, "init-tools.cmd")] + else: + command = ["bash", os.path.join(coreclr_repo_location, "init-tools.sh")] + + print(" ".join(command)) + subprocess.check_output(command) + + setup = True + + return setup + +def setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root): + """ Setup CoreDisTools if needed + + Args: + coreclr_repo_location(str) : coreclr repo location + host_os(str) : os + arch(str) : arch + core_root(str) : core_root + """ + + if host_os.lower() == "osx": + print("GCStress C is not supported on your platform.") + sys.exit(1) + + unsupported_arches = ["arm", "arm64"] + + if arch in unsupported_arches: + # Nothing to do; CoreDisTools unneeded. + return + + command = None + test_location = os.path.join(coreclr_repo_location, "tests") + if host_os == "Windows_NT": + command = [os.path.join(test_location, "setup-stress-dependencies.cmd"), "/arch", arch, "/outputdir", core_root] + else: + command = [os.path.join(test_location, "setup-stress-dependencies.sh"), "--outputDir=%s" % core_root] + + sys.stdout.flush() # flush output before creating sub-process + proc = subprocess.Popen(command) + proc.communicate() + + if proc.returncode != 0: + print("Failed to set up stress dependencies.") + sys.exit(1) + +def precompile_core_root(test_location, + host_os, + arch, + core_root, + use_jit_disasm=False, + altjit_name=False): + """ Precompile all of the assemblies in the core_root directory + + Args: + test_location(str) : test location + host_os(str) : os + core_root(str) : location of core_root + use_jit_disasm(Bool) : use jit disasm + altjit_name(str) : name of the altjit + + """ + + skip_list = [ + ".*xunit.*", + ".*api-ms-win-core.*", + ".*api-ms-win.*", + ".*System.Private.CoreLib.*" + ] + + unix_skip_list = [ + ".*mscorlib.*", + ".*System.Runtime.WindowsRuntime.*", + ".*System.Runtime.WindowsRuntime.UI.Xaml.*", + ".*R2RDump.dll.*" + ] + + arm64_unix_skip_list = [ + ".*Microsoft.CodeAnalysis.VisualBasic.*", + ".*System.Net.NameResolution.*", + ".*System.Net.Sockets.*", + ".*System.Net.Primitives.*" + ] + + if host_os != "Windows_NT": + skip_list += unix_skip_list + + if arch == "arm64": + skip_list += arm64_unix_skip_list + + assert os.path.isdir(test_location) + assert os.path.isdir(core_root) + + crossgen = os.path.join(core_root, "crossgen%s" % (".exe" if host_os == "Windows_NT" else "")) + assert os.path.isfile(crossgen) + + def call_crossgen(file, env): + assert os.path.isfile(crossgen) + command = [crossgen, "/Platform_Assemblies_Paths", core_root, file] + + if use_jit_disasm: + core_run = os.path.join(core_root, "corerun%s" % (".exe" if host_os == "Windows_NT" else "")) + assert os.path.isfile(core_run) + + command = [core_run, + os.path.join(core_root, "jit-dasm.dll"), + "--crossgen", + crossgen, + "--platform", + core_root, + "--output", + os.path.join(test_location, "dasm"), + file] + + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + proc.communicate() + + return_code = proc.returncode + + if return_code == -2146230517: + print("%s is not a managed assembly." % file) + return False + + if return_code != 0: + print("Unable to precompile %s (%d)" % (file, return_code)) + return False + + print("Successfully precompiled %s" % file) + return True + + print("Precompiling all assemblies in %s" % core_root) + print("") + + env = os.environ.copy() + + if not altjit_name is None: + env["COMPlus_AltJit"]="*" + env["COMPlus_AltJitNgen"]="*" + env["COMPlus_AltJitName"]=altjit_name + env["COMPlus_AltJitAssertOnNYI"]="1" + env["COMPlus_NoGuiOnAssert"]="1" + env["COMPlus_ContinueOnAssert"]="0" + + dlls = [os.path.join(core_root, item) for item in os.listdir(core_root) if item.endswith("dll") and "mscorlib" not in item] + + def in_skip_list(item): + found = False + for skip_re in skip_list: + if re.match(skip_re, item.lower()) is not None: + found = True + return found + + dlls = [dll for dll in dlls if not in_skip_list(dll)] + + for dll in dlls: + call_crossgen(dll, env) + + print("") + +def setup_core_root(host_os, + arch, + build_type, + coreclr_repo_location, + test_native_bin_location, + product_location, + test_location, + core_root, + is_corefx=False, + generate_layout=True): + """ Setup the core root + + Args: + host_os(str) : os + arch(str) : architecture + build_type(str) : build configuration + coreclr_repo_location(str) : coreclr repo location + product_location(str) : Product location + core_root(str) : Location for core_root + is_corefx : Building corefx core_root + + Optional Args: + is_corefx(Bool) : Pass if planning on running corex + : tests + + """ + global g_verbose + + assert os.path.isdir(product_location) + + # Create core_root if it does not exist + if os.path.isdir(core_root): + shutil.rmtree(core_root) + + os.makedirs(core_root) + + # Setup the dotnetcli location + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + # Set global env variables. + os.environ["__BuildLogRootName"] = "Restore_Product" + + if host_os != "Windows_NT": + os.environ["__DistroRid"] = "%s-%s" % ("osx" if sys.platform == "darwin" else "linux", arch) + + command = [os.path.join(coreclr_repo_location, "run.%s" % ("cmd" if host_os == "Windows_NT" else "sh")), + "build", + "-Project=%s" % os.path.join(coreclr_repo_location, "tests", "build.proj")] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + log_path = os.path.join(logs_dir, "Restore_Product%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + msbuild_log_params = "/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log + msbuild_wrn_params = "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log + msbuild_err_params = "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log + + command += ["-MsBuildLog=%s" % msbuild_log_params, + "-MsBuildWrn=%s" % msbuild_wrn_params, + "-MsBuildErr=%s" % msbuild_err_params] + + if host_os != "Windows_NT": + command = ["bash"] + command + command += ["-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + + if g_verbose: + command += ["-verbose"] + + command += [ "-BatchRestorePackages", + "-BuildType=%s" % build_type, + "-BuildArch=%s" % arch, + "-BuildOS=%s" % host_os] + + print("Restoring packages...") + print(" ".join(command)) + + sys.stdout.flush() # flush output before creating sub-process + if not g_verbose: + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + proc = subprocess.Popen(command) + + try: + proc.communicate() + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + + if proc.returncode == 1: + print("Error: package restore failed.") + return False + + os.environ["__BuildLogRootName"] = "" + + # Copy restored packages to core_root + # Set global env variables. + os.environ["__BuildLogRootName"] = "Tests_Overlay_Managed" + + if host_os != "Windows_NT": + os.environ["__DistroRid"] = "%s-%s" % ("osx" if sys.platform == "darwin" else "linux", arch) + os.environ["__RuntimeId"] = os.environ["__DistroRid"] + + os.environ["Core_Root"] = core_root + os.environ["xUnitTestBinBase"] = os.path.dirname(os.path.dirname(core_root)) + + command = [os.path.join(coreclr_repo_location, "run.%s" % ("cmd" if host_os == "Windows_NT" else "sh")), + "build", + "-Project=%s" % os.path.join(coreclr_repo_location, "tests", "runtest.proj")] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + log_path = os.path.join(logs_dir, "Tests_Overlay_Managed%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + msbuild_log_params = "/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log + msbuild_wrn_params = "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log + msbuild_err_params = "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log + + command += ["-MsBuildLog=%s" % msbuild_log_params, + "-MsBuildWrn=%s" % msbuild_wrn_params, + "-MsBuildErr=%s" % msbuild_err_params] + + if host_os != "Windows_NT": + command = ["bash"] + command + command += ["-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + + if g_verbose: + command += ["-verbose"] + + command += [ "-testOverlay", + "-BuildType=%s" % build_type, + "-BuildArch=%s" % arch, + "-BuildOS=%s" % host_os] + + print("") + print("Creating Core_Root...") + print(" ".join(command)) + + sys.stdout.flush() # flush output before creating sub-process + if not g_verbose: + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + proc = subprocess.Popen(command) + + try: + proc.communicate() + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + + if proc.returncode == 1: + print("Error: creating Core_Root failed.") + return False + + os.environ["__BuildLogRootName"] = "" + os.environ["xUnitTestBinBase"] = "" + os.environ["__RuntimeId"] = "" + + def copy_tree(src, dest): + """ Simple copy from src to dest + """ + assert os.path.isdir(src) + assert os.path.isdir(dest) + + for item in os.listdir(src): + if ".nuget" in item: + pass + item = os.path.join(src, item) + + if os.path.isfile(item): + shutil.copy2(item, dest) + + if host_os != "Windows_NT": + # Set executable bit + os.chmod(os.path.join(dest, item), 0o774) + else: + new_dir = os.path.join(dest, os.path.basename(item)) + if os.path.isdir(new_dir): + shutil.rmtree(new_dir) + + shutil.copytree(item, new_dir) + + # Copy the product dir to the core_root directory + print("") + print("Copying Product Bin to Core_Root:") + print("cp -r %s%s* %s" % (product_location, os.path.sep, core_root)) + copy_tree(product_location, core_root) + print("---------------------------------------------------------------------") + print("") + + if is_corefx: + corefx_utility_setup = os.path.join(coreclr_repo_location, + "src", + "Common", + "CoreFX", + "TestFileSetup", + "CoreFX.TestUtils.TestFileSetup.csproj") + + os.environ["__BuildLogRootName"] = "Tests_GenerateTestHost" + msbuild_command = [dotnetcli_location, + "msbuild", + os.path.join(coreclr_repo_location, "tests", "runtest.proj"), + "/p:GenerateRuntimeLayout=true"] + + sys.stdout.flush() # flush output before creating sub-process + proc = subprocess.Popen(msbuild_command) + proc.communicate() + + if not proc.returncode == 0: + print("Error: generating test host failed.") + return False + + os.environ["__BuildLogRootName"] = "" + + msbuild_command = [dotnetcli_location, + "msbuild", + "/t:Restore", + corefx_utility_setup] + + sys.stdout.flush() # flush output before creating sub-process + proc = subprocess.Popen(msbuild_command) + proc.communicate() + + if proc.returncode == 1: + print("Error: msbuild failed.") + return False + + corefx_logpath = os.path.join(coreclr_repo_location, + "bin", + "tests", + "%s.%s.%s" % (host_os, arch, build_type), + "CoreFX", + "CoreFXTestUtilities") + + msbuild_command = [dotnetcli_location, + "msbuild", + "/p:Configuration=%s" % build_type, + "/p:OSGroup=%s" % host_os, + "/p:Platform=%s" % arch, + "/p:OutputPath=%s" % corefx_logpath, + corefx_utility_setup] + + sys.stdout.flush() # flush output before creating sub-process + proc = subprocess.Popen(msbuild_command) + proc.communicate() + + if proc.returncode == 1: + print("Error: msbuild failed.") + return False + + print("Core_Root setup.") + print("") + + return True + +def delete_existing_wrappers(test_location): + """ Delete the existing xunit wrappers + + Args: + test_location(str) : location of the test + """ + + assert os.path.isdir(test_location) or os.path.isfile(test_location) + + extension = "dll" + + if os.path.isdir(test_location): + for item in os.listdir(test_location): + delete_existing_wrappers(os.path.join(test_location, item)) + elif test_location.endswith(extension) and "xunitwrapper" in test_location.lower(): + # Delete the test wrapper. + + print("rm %s" % test_location) + os.remove(test_location) + +def build_test_wrappers(host_os, + arch, + build_type, + coreclr_repo_location, + test_location): + """ Build the coreclr test wrappers + + Args: + host_os(str) : os + arch(str) : architecture + build_type(str) : build configuration + coreclr_repo_location(str) : coreclr repo location + test_location(str) : location of the test + + Notes: + Build the xUnit test wrappers. Note that this will have been done as a + part of build-test.cmd/sh. It is possible that the host has a different + set of dependencies from the target or the exclude list has changed + after building. + + """ + global g_verbose + + delete_existing_wrappers(test_location) + + # Setup the dotnetcli location + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + # Set global env variables. + os.environ["__BuildLogRootName"] = "Tests_XunitWrapper" + os.environ["__Exclude"] = os.path.join(coreclr_repo_location, "tests", "issues.targets") + + command = [dotnetcli_location, + "msbuild", + os.path.join(coreclr_repo_location, "tests", "runtest.proj"), + "/p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/", + "/p:BuildWrappers=true", + "/p:TargetsWindows=%s" % ("true" if host_os == "Windows_NT" else "false")] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + log_path = os.path.join(logs_dir, "Tests_XunitWrapper%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + command += ["/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log, + "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log, + "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log, + "/consoleloggerparameters:Summary"] + + command += ["/p:__BuildOS=%s" % host_os, + "/p:__BuildArch=%s" % arch, + "/p:__BuildType=%s" % build_type, + "/p:__LogsDir=%s" % logs_dir] + + print("Creating test wrappers...") + print(" ".join(command)) + + sys.stdout.flush() # flush output before creating sub-process + if not g_verbose: + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if not running_in_ci(): + try: + expected_time_to_complete = 60*5 # 5 Minutes + estimated_time_running = 0 + + time_delta = 1 + + while True: + time_remaining = expected_time_to_complete - estimated_time_running + time_in_minutes = math.floor(time_remaining / 60) + remaining_seconds = time_remaining % 60 + + sys.stdout.write("\rEstimated time remaining: %d minutes %d seconds" % (time_in_minutes, remaining_seconds)) + sys.stdout.flush() + + time.sleep(time_delta) + estimated_time_running += time_delta + + if estimated_time_running == expected_time_to_complete: + break + if proc.poll() is not None: + break + + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + else: + proc = subprocess.Popen(command) + + try: + proc.communicate() + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + + if proc.returncode == 1: + print("Error: creating test wrappers failed.") + return False + +def find_test_from_name(host_os, test_location, test_name): + """ Given a test's name return the location on disk + + Args: + host_os (str) : os + test_location (str) :path to the coreclr tests + test_name (str) : Name of the test, all special characters will have + : been replaced with underscores. + + Return: + test_path (str): Path of the test based on its name + """ + + location = test_name + + # Lambdas and helpers + is_file_or_dir = lambda path : os.path.isdir(path) or os.path.isfile(path) + def match_filename(test_path): + # Scan through the test directory looking for a similar + # file + global file_name_cache + + if not os.path.isdir(os.path.dirname(test_path)): + pass + + assert os.path.isdir(os.path.dirname(test_path)) + size_of_largest_name_file = 0 + + dir_contents = file_name_cache[os.path.dirname(test_path)] + + if dir_contents is None: + dir_contents = defaultdict(lambda: None) + for item in os.listdir(os.path.dirname(test_path)): + dir_contents[re.sub("[%s]" % string.punctuation, "_", item)] = item + + file_name_cache[os.path.dirname(test_path)] = dir_contents + + # It is possible there has already been a match + # therefore we need to remove the punctuation again. + basename_to_match = re.sub("[%s]" % string.punctuation, "_", os.path.basename(test_path)) + if basename_to_match in dir_contents: + test_path = os.path.join(os.path.dirname(test_path), dir_contents[basename_to_match]) + + size_of_largest_name_file = len(max(dir_contents, key=len)) + + return test_path, size_of_largest_name_file + + def dir_has_nested_substrings(test_path, test_item): + """ A directory has multiple paths where one path is a substring of another + """ + + dir_contents = file_name_cache[os.path.dirname(test_path)] + + if dir_contents is None: + dir_contents = defaultdict(lambda: None) + for item in os.listdir(os.path.dirname(test_path)): + dir_contents[re.sub("[%s]" % string.punctuation, "_", item)] = item + + file_name_cache[os.path.dirname(test_path)] = dir_contents + + test_item = re.sub("[%s]" % string.punctuation, "_", test_item) + + count = 0 + for item in dir_contents: + if test_item in item: + count += 1 + + return count > 1 + + # Find the test by searching down the directory list. + starting_path = test_location + loc_split = location.split("_") + append = False + for index, item in enumerate(loc_split): + if not append: + test_path = os.path.join(starting_path, item) + else: + append = False + test_path, size_of_largest_name_file = match_filename(starting_path + "_" + item) + + if not is_file_or_dir(test_path): + append = True + + # It is possible that there is another directory that is named + # without an underscore. + elif index + 1 < len(loc_split) and os.path.isdir(test_path): + next_test_path = os.path.join(test_path, loc_split[index + 1]) + + if not is_file_or_dir(next_test_path) or dir_has_nested_substrings(test_path, item): + added_path = test_path + for forward_index in range(index + 1, len(loc_split)): + added_path, size_of_largest_name_file = match_filename(added_path + "_" + loc_split[forward_index]) + if is_file_or_dir(added_path): + append = True + break + elif size_of_largest_name_file < len(os.path.basename(added_path)): + break + + starting_path = test_path + + location = starting_path + if not os.path.isfile(location): + pass + + assert(os.path.isfile(location)) + + return location + +def parse_test_results(host_os, arch, build_type, coreclr_repo_location, test_location): + """ Parse the test results for test execution information + + Args: + host_os : os + arch : architecture run on + build_type : build configuration (debug, checked, release) + coreclr_repo_location : coreclr repo location + test_location : path to coreclr tests + + """ + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + log_path = os.path.join(logs_dir, "TestRunResults_%s_%s_%s" % (host_os, arch, build_type)) + print("Parsing test results from (%s)" % log_path) + + test_run_location = os.path.join(coreclr_repo_location, "bin", "Logs", "testRun.xml") + + if not os.path.isfile(test_run_location): + # Check if this is a casing issue + + found = False + for item in os.listdir(os.path.dirname(test_run_location)): + item_lower = item.lower() + if item_lower == "testrun.xml": + # Correct the name. + os.rename(os.path.join(coreclr_repo_location, "bin", "Logs", item), test_run_location) + found = True + break + + if not found: + print("Unable to find testRun.xml. This normally means the tests did not run.") + print("It could also mean there was a problem logging. Please run the tests again.") + + return + + if host_os != "Windows_NT" and running_in_ci(): + # Huge hack. + # TODO change netci to parse testRun.xml + shutil.copy2(test_run_location, os.path.join(os.path.dirname(test_run_location), "coreclrtests.xml")) + + assemblies = xml.etree.ElementTree.parse(test_run_location).getroot() + + tests = defaultdict(lambda: None) + for assembly in assemblies: + for collection in assembly: + if collection.tag == "errors" and collection.text != None: + # Something went wrong during running the tests. + print("Error running the tests, please run runtest.py again.") + sys.exit(1) + elif collection.tag != "errors": + test_name = None + for test in collection: + type = test.attrib["type"] + method = test.attrib["method"] + + type = type.split("._")[0] + test_name = type + method + + assert test_name != None + + failed = collection.attrib["failed"] + skipped = collection.attrib["skipped"] + passed = collection.attrib["passed"] + time = float(collection.attrib["time"]) + + test_output = None + + if failed == "1": + failure_info = collection[0][0] + + test_output = failure_info[0].text + + test_location_on_filesystem = find_test_from_name(host_os, test_location, test_name) + + assert os.path.isfile(test_location_on_filesystem) + + assert tests[test_name] == None + tests[test_name] = defaultdict(lambda: None, { + "name": test_name, + "test_path": test_location_on_filesystem, + "failed": failed, + "skipped": skipped, + "passed": passed, + "time": time, + "test_output": test_output + }) + + return tests + +def print_summary(tests): + """ Print a summary of the test results + + Args: + tests (defaultdict[String]: { }): The tests that were reported by + : xunit + + """ + + assert tests is not None + + failed_tests = [] + passed_tests = [] + skipped_tests = [] + + for test in tests: + test = tests[test] + + if test["failed"] == "1": + failed_tests.append(test) + elif test["passed"] == "1": + passed_tests.append(test) + else: + skipped_tests.append(test) + + failed_tests.sort(key=lambda item: item["time"], reverse=True) + passed_tests.sort(key=lambda item: item["time"], reverse=True) + skipped_tests.sort(key=lambda item: item["time"], reverse=True) + + def print_tests_helper(tests, stop_count): + for index, item in enumerate(tests): + time = item["time"] + unit = "seconds" + time_remainder = "" + second_unit = "" + saved_time = time + remainder_str = "" + + # If it can be expressed in hours + if time > 60**2: + time = saved_time / (60**2) + time_remainder = saved_time % (60**2) + time_remainder /= 60 + time_remainder = math.floor(time_remainder) + unit = "hours" + second_unit = "minutes" + + remainder_str = " %s %s" % (int(time_remainder), second_unit) + + elif time > 60 and time < 60**2: + time = saved_time / 60 + time_remainder = saved_time % 60 + time_remainder = math.floor(time_remainder) + unit = "minutes" + second_unit = "seconds" + + remainder_str = " %s %s" % (int(time_remainder), second_unit) + + print("%s (%d %s%s)" % (item["test_path"], time, unit, remainder_str)) + + if stop_count != None: + if index >= stop_count: + break + + if len(failed_tests) > 0: + print("%d failed tests:" % len(failed_tests)) + print("") + print_tests_helper(failed_tests, None) + + # The following code is currently disabled, as it produces too much verbosity in a normal + # test run. It could be put under a switch, or else just enabled as needed when investigating + # test slowness. + # + # if len(passed_tests) > 50: + # print("") + # print("50 slowest passing tests:") + # print("") + # print_tests_helper(passed_tests, 50) + + if len(failed_tests) > 0: + print("") + print("#################################################################") + print("Output of failing tests:") + print("") + + for item in failed_tests: + print("[%s]: " % item["test_path"]) + print("") + + test_output = item["test_output"] + + # XUnit results are captured as escaped characters. + test_output = test_output.replace("\\r", "\r") + test_output = test_output.replace("\\n", "\n") + + print(test_output) + test_output = test_output.replace("/r", "\r") + test_output = test_output.replace("/n", "\n") + unicode_output = None + if sys.version_info < (3,0): + # Handle unicode characters in output in python2.* + unicode_output = unicode(test_output, "utf-8") + else: + unicode_output = test_output + + print(unicode_output) + print("") + + print("") + print("#################################################################") + print("End of output of failing tests") + print("#################################################################") + print("") + + print("") + print("Total tests run : %d" % len(tests)) + print("Total passing tests: %d" % len(passed_tests)) + print("Total failed tests : %d" % len(failed_tests)) + print("Total skipped tests: %d" % len(skipped_tests)) + print("") + +def create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_location, tests): + """ Go through the failing tests and create repros for them + + Args: + host_os (String) : os + arch (String) : architecture + build_type (String) : build configuration (debug, checked, release) + core_root (String) : Core_Root path + coreclr_repo_location (String) : Location of coreclr git repo + tests (defaultdict[String]: { }): The tests that were reported by + : xunit + + """ + assert tests is not None + + failed_tests = [tests[item] for item in tests if tests[item]["failed"] == "1"] + if len(failed_tests) == 0: + return + + bin_location = os.path.join(coreclr_repo_location, "bin") + assert os.path.isdir(bin_location) + + repro_location = os.path.join(bin_location, "repro", "%s.%s.%s" % (host_os, arch, build_type)) + if os.path.isdir(repro_location): + shutil.rmtree(repro_location) + + print("") + print("Creating repro files at: %s" % repro_location) + + os.makedirs(repro_location) + assert os.path.isdir(repro_location) + + # Now that the repro_location exists under /bin/repro + # create wrappers which will simply run the test with the correct environment + for test in failed_tests: + debug_env = DebugEnv(host_os, arch, build_type, env, core_root, coreclr_repo_location, test) + debug_env.write_repro() + + print("Repro files written.") + +def do_setup(host_os, + arch, + build_type, + coreclr_repo_location, + product_location, + test_location, + test_native_bin_location, + core_root, + unprocessed_args, + test_env): + global gc_stress_c + + # Setup the tools for the repo. + setup_tools(host_os, coreclr_repo_location) + + if unprocessed_args.generate_layout: + success = setup_core_root(host_os, + arch, + build_type, + coreclr_repo_location, + test_native_bin_location, + product_location, + test_location, + core_root) + + if not success: + print("Error: GenerateLayout failed.") + sys.exit(1) + + if unprocessed_args.generate_layout_only: + sys.exit(0) + + if unprocessed_args.precompile_core_root: + precompile_core_root(test_location, host_os, arch, core_root, use_jit_disasm=args.jitdisasm, altjit_name=unprocessed_args.crossgen_altjit) + + # If COMPlus_GCStress is set then we need to setup cordistools + if gc_stress_c: + setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root) + + build_info = None + is_same_os = None + is_same_arch = None + is_same_build_type = None + + # We will write out build information into the test directory. This is used + # by runtest.py to determine whether we need to rebuild the test wrappers. + if os.path.isfile(os.path.join(test_location, "build_info.json")): + with open(os.path.join(test_location, "build_info.json")) as file_handle: + build_info = json.load(file_handle) + is_same_os = build_info["build_os"] == host_os + is_same_arch = build_info["build_arch"] == arch + is_same_build_type = build_info["build_type"] == build_type + + # Copy all the native libs to core_root + if host_os != "Windows_NT" and not (is_same_os and is_same_arch and is_same_build_type): + copy_native_test_bin_to_core_root(host_os, os.path.join(test_native_bin_location, "src"), core_root) + + # Line ending only need to be corrected if this is a cross build. + correct_line_endings(host_os, test_location) + + if unprocessed_args.build_test_wrappers: + build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) + elif build_info is None: + build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) + elif not (is_same_os and is_same_arch and is_same_build_type): + build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) + + return run_tests(host_os, + arch, + build_type, + core_root, + coreclr_repo_location, + test_location, + test_native_bin_location, + is_illink=unprocessed_args.il_link, + is_long_gc=unprocessed_args.long_gc, + is_gcsimulator=unprocessed_args.gcsimulator, + is_jitdasm=unprocessed_args.jitdisasm, + is_ilasm=unprocessed_args.ilasmroundtrip, + run_sequential=unprocessed_args.sequential, + run_crossgen_tests=unprocessed_args.run_crossgen_tests, + test_env=test_env) + +################################################################################ +# Main +################################################################################ + +def main(args): + global g_verbose + g_verbose = args.verbose + + host_os, arch, build_type, coreclr_repo_location, product_location, core_root, test_location, test_native_bin_location = setup_args(args) + + ret_code = 0 + + env = get_environment(test_env=args.test_env) + if not args.analyze_results_only: + if args.test_env is not None: + ret_code = do_setup(host_os, + arch, + build_type, + coreclr_repo_location, + product_location, + test_location, + test_native_bin_location, + core_root, + args, + args.test_env) + else: + ret_code = create_and_use_test_env(host_os, + env, + lambda path: do_setup(host_os, + arch, + build_type, + coreclr_repo_location, + product_location, + test_location, + test_native_bin_location, + core_root, + args, + path)) + print("Test run finished.") + + tests = parse_test_results(host_os, arch, build_type, coreclr_repo_location, test_location) + + if tests is not None: + print_summary(tests) + create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_location, tests) + + return ret_code + +################################################################################ +# __main__ +################################################################################ + +if __name__ == "__main__": + args = parser.parse_args() + sys.exit(main(args)) diff --git a/tests/scripts/scripts.csproj b/tests/scripts/scripts.csproj index b1a1aab6e54e..e803a60a37b0 100644 --- a/tests/scripts/scripts.csproj +++ b/tests/scripts/scripts.csproj @@ -14,8 +14,8 @@ 1.0.0-alpha-build0040 - - 1.0.2-prerelease-00177 + + $(XunitPackageVersion) 1.0.0-prerelease-00629-04 diff --git a/tests/src/CLRTest.GC.targets b/tests/src/CLRTest.GC.targets index f959aaf6e006..f28eca4b99d0 100644 --- a/tests/src/CLRTest.GC.targets +++ b/tests/src/CLRTest.GC.targets @@ -18,10 +18,7 @@ WARNING: When setting properties based on their current state (for example: --> - - 2 - 0 - + 0 - - + + + + + + - Debug - AnyCPU - CoreclrTestWrapperLib - 2.0 - {95DFC527-4DC1-495E-97D7-E94EE1F7140D} - Library - {786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} - ..\..\ - BuildOnly - false - BuildOnly true - - - - - - - - False - - + - - - - + + + diff --git a/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs b/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs index e431f3a3337d..c946e2e6a615 100644 --- a/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs +++ b/tests/src/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs @@ -97,7 +97,6 @@ public class CoreclrTestWrapperLib // Default timeout set to 10 minutes public const int DEFAULT_TIMEOUT = 1000 * 60*10; - public const string GC_STRESS_LEVEL = "__GCSTRESSLEVEL"; public const string COLLECT_DUMPS_ENVIRONMENT_VAR = "__CollectDumps"; public const string CRASH_DUMP_FOLDER_ENVIRONMENT_VAR = "__CrashDumpFolder"; @@ -167,8 +166,6 @@ public int RunTest(string executable, string outputFile, string errorFile) string environmentVar = Environment.GetEnvironmentVariable(TIMEOUT_ENVIRONMENT_VAR); int timeout = environmentVar != null ? int.Parse(environmentVar) : DEFAULT_TIMEOUT; - string gcstressVar = Environment.GetEnvironmentVariable(GC_STRESS_LEVEL); - // Check if we are running in Windows string operatingSystem = System.Environment.GetEnvironmentVariable("OS"); bool runningInWindows = (operatingSystem != null && operatingSystem.StartsWith("Windows")); @@ -183,13 +180,6 @@ public int RunTest(string executable, string outputFile, string errorFile) using (var errorWriter = new StreamWriter(errorStream)) using (Process process = new Process()) { - if (gcstressVar!=null) - { - //Note: this is not the best way to set the Env, but since we are using - //Desktop to start the tests, this Env will affect the test harness behavior - process.StartInfo.EnvironmentVariables["COMPlus_GCStress"] = gcstressVar; - } - // Windows can run the executable implicitly if (runningInWindows) { diff --git a/tests/src/Common/Desktop.Coreclr.TestWrapper/Desktop.Coreclr.TestWrapper.csproj b/tests/src/Common/Desktop.Coreclr.TestWrapper/Desktop.Coreclr.TestWrapper.csproj deleted file mode 100644 index b26c89370ba0..000000000000 --- a/tests/src/Common/Desktop.Coreclr.TestWrapper/Desktop.Coreclr.TestWrapper.csproj +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - Debug - AnyCPU - {5259DD45-EA34-4EE7-B5C5-430B80384544} - Library - Coreclr.TestWrapper - Coreclr.TestWrapper - .NETFramework - v4.5 - true - BuildOnly - true - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - diff --git a/tests/src/Common/external/external.depproj b/tests/src/Common/external/external.depproj index 46d753cdf019..aedcfe01fefb 100644 --- a/tests/src/Common/external/external.depproj +++ b/tests/src/Common/external/external.depproj @@ -12,8 +12,8 @@ win7-x64 true $(TargetingPackPath) - xunit.console.netcore - xunit.runner.console + xunit.runner.console + netcoreapp1.0 SharedLibrary false @@ -45,8 +45,8 @@ $(XunitPackageVersion) - - $(XunitConsoleNetcorePackageVersion) + + $(XunitPackageVersion) $(XunitPackageVersion) @@ -75,7 +75,6 @@ - @@ -84,14 +83,14 @@ $(SourceDir)Common\external\obj\project.assets.json - - + false $(XUnitRunnerPackageId) @@ -99,4 +98,4 @@ - \ No newline at end of file + diff --git a/tests/src/Common/vc_runtime/vc_runtime.csproj b/tests/src/Common/vc_runtime/vc_runtime.csproj index 3380240ff690..7a51486d0fc0 100644 --- a/tests/src/Common/vc_runtime/vc_runtime.csproj +++ b/tests/src/Common/vc_runtime/vc_runtime.csproj @@ -5,8 +5,8 @@ Debug AnyCPU BuildOnly - .NETCoreApp,Version=v1.1 - netcoreapp1.1 + .NETCoreApp,Version=v2.2 + netcoreapp2.2 false @@ -15,10 +15,10 @@ - netcoreapp1.1 + netcoreapp2.2 .NETCoreApp $(PackageTargetFallback);dnxcore50;netcoreapp1.1;portable-net45+win8 - win-arm + win-arm;win-arm64 true false diff --git a/tests/src/JIT/Performance/CodeQuality/Layout/SearchLoops.cs b/tests/src/JIT/Performance/CodeQuality/Layout/SearchLoops.cs index 60684f65df21..1d2bdaf88b6a 100644 --- a/tests/src/JIT/Performance/CodeQuality/Layout/SearchLoops.cs +++ b/tests/src/JIT/Performance/CodeQuality/Layout/SearchLoops.cs @@ -47,13 +47,13 @@ public SearchLoops() } [Benchmark(InnerIterationCount = 20000000)] - public void LoopReturn() + public void LoopReturnIter() { Benchmark.Iterate(() => LoopReturn(test1, test2)); } [Benchmark(InnerIterationCount = 20000000)] - public void LoopGoto() + public void LoopGotoIter() { Benchmark.Iterate(() => LoopGoto(test1, test2)); } diff --git a/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs b/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs index bfa1f43a5ff1..ed7ec6157ea8 100644 --- a/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs +++ b/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs @@ -127,7 +127,7 @@ static bool Test(int index, bool isXunitBenchmark) // Entrypoint for xunit-perf to call the benchmark [Benchmark] [MemberData(nameof(ArrayedBoxedIndicesToTest))] - public static bool Test(object boxedIndex) + public static bool TestWithXunit(object boxedIndex) { return Test((int)boxedIndex, true); } diff --git a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj index 563a9a708704..881d84ff55c0 100644 --- a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj +++ b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj @@ -82,8 +82,8 @@ $(XunitPackageVersion) - - $(XunitConsoleNetcorePackageVersion) + + $(XunitPackageVersion) $(XunitPackageVersion) diff --git a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj index d554c165f6b4..d2789e5957ad 100644 --- a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj +++ b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj @@ -79,8 +79,8 @@ $(XunitPackageVersion) - - $(XunitConsoleNetcorePackageVersion) + + $(XunitPackageVersion) $(XunitPackageVersion) diff --git a/tests/src/JIT/config/benchmark/benchmark.csproj b/tests/src/JIT/config/benchmark/benchmark.csproj index 4d49fdee3ead..aecfdec2773e 100644 --- a/tests/src/JIT/config/benchmark/benchmark.csproj +++ b/tests/src/JIT/config/benchmark/benchmark.csproj @@ -97,8 +97,8 @@ $(XunitPackageVersion) - - $(XunitConsoleNetcorePackageVersion) + + $(XunitPackageVersion) $(XunitPackageVersion) diff --git a/tests/src/TestWrappersConfig/TestWrappersConfig.csproj b/tests/src/TestWrappersConfig/TestWrappersConfig.csproj deleted file mode 100644 index 76128421d2c9..000000000000 --- a/tests/src/TestWrappersConfig/TestWrappersConfig.csproj +++ /dev/null @@ -1,38 +0,0 @@ - - - - - Debug - AnyCPU - BuildOnly - false - - - - $(MicrosoftNETCorePlatformsPackageVersion) - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - $(XunitPackageVersion) - - - - netcoreapp1.1;net45 - .NETCoreApp - $(PackageTargetFallback);dnxcore50;portable-net45+win8 - $(TargetRid) - true - false - - - - \ No newline at end of file diff --git a/tests/src/dir.common.props b/tests/src/dir.common.props deleted file mode 100644 index dda175574c91..000000000000 --- a/tests/src/dir.common.props +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - <__BuildArch Condition="'$(__BuildArch)' == ''">x64 - <__BuildType Condition="'$(__BuildType)' == ''">Debug - <__BuildOS Condition="'$(__BuildOS)' == ''">Windows_NT - $(__BuildArch) - $(__BuildType) - $(__BuildOS) - $(BuildType) - $(BuildArch) - - - - - true - false - full - $(DefineConstants);DEBUG;TRACE;XUNIT_PERF - - - true - true - pdbonly - $(DefineConstants);TRACE;XUNIT_PERF - - - true - true - full - $(DefineConstants);DEBUG;TRACE;XUNIT_PERF - - - - - 64 - 64 - 32 - 32 - - - - - $(BuildOS).$(Platform).$(Configuration) - $(ProjectDir)\..\bin\tests - $(__TestRootDir) - $(BaseOutputPath)\$(OSPlatformConfig)\ - $(BaseOutputPathWithConfig) - $(ProjectDir)\..\bin\tests\obj\$(OSPlatformConfig)\Managed\ - $(__ManagedTestIntermediatesDir)\ - <__NativeTestIntermediatesDir Condition="'$(__NativeTestIntermediatesDir)' == ''">$([System.IO.Path]::GetFullPath($(BaseOutputPathWithConfig)..\obj\$(BuildOS).$(Platform).$(Configuration)\Native\)) - $(MSBuildProjectName)\ - $([System.String]::Copy('$(MSBuildProjectDirectory)').Replace($(SourceDir),''))\$(MSBuildProjectName) - $(BaseIntermediateOutputPath)$(BuildProjectRelativeDir)\ - $(BaseOutputPathWithConfig)$(BuildProjectRelativeDir)\ - $(BaseOutputPath)\testStagingDir\ - $(TestWorkingDir)$(OSPlatformConfig)\$(MSBuildProjectName)/ - - - - diff --git a/tests/src/dir.props b/tests/src/dir.props index abdb7cc0f574..e5d652ba715d 100644 --- a/tests/src/dir.props +++ b/tests/src/dir.props @@ -1,5 +1,52 @@ - + + + + + + + true + false + full + $(DefineConstants);DEBUG;TRACE;XUNIT_PERF + + + true + true + pdbonly + $(DefineConstants);TRACE;XUNIT_PERF + + + true + true + full + $(DefineConstants);DEBUG;TRACE;XUNIT_PERF + + + + + 64 + 64 + 32 + 32 + + + + + $(ProjectDir)..\bin\tests + $(__TestRootDir) + $(BaseOutputPath)\$(OSPlatformConfig)\ + $(BaseOutputPathWithConfig) + $(ProjectDir)\..\bin\tests\obj\$(OSPlatformConfig)\Managed\ + $(__ManagedTestIntermediatesDir)\ + <__NativeTestIntermediatesDir Condition="'$(__NativeTestIntermediatesDir)' == ''">$([System.IO.Path]::GetFullPath($(BaseOutputPathWithConfig)..\obj\$(BuildOS).$(Platform).$(Configuration)\Native\)) + $(MSBuildProjectName)\ + $([System.String]::Copy('$(MSBuildProjectDirectory)').Replace($(SourceDir),''))\$(MSBuildProjectName) + $(BaseIntermediateOutputPath)$(BuildProjectRelativeDir)\ + $(BaseOutputPathWithConfig)$(BuildProjectRelativeDir)\ + $(BaseOutputPath)\testStagingDir\ + $(TestWorkingDir)$(OSPlatformConfig)\$(MSBuildProjectName)/ + @@ -11,8 +58,6 @@ 78,162,164,168,169,219,251,252,414,429,642,649,652,675,1691,1717,1718,3001,3002,3003,3005,3008 false true - - $(MSBuildThisFileDirectory)TestWrappersConfig\ Test diff --git a/tests/src/dirs.proj b/tests/src/dirs.proj index 6009e2ed080e..3a54541043b0 100644 --- a/tests/src/dirs.proj +++ b/tests/src/dirs.proj @@ -16,8 +16,7 @@ - - + diff --git a/tests/src/performance/performance.csproj b/tests/src/performance/performance.csproj index 94caf35ac235..c6c0b5c98dd1 100644 --- a/tests/src/performance/performance.csproj +++ b/tests/src/performance/performance.csproj @@ -94,8 +94,8 @@ $(XunitPackageVersion) - - $(XunitConsoleNetcorePackageVersion) + + $(XunitPackageVersion) $(XunitPackageVersion) diff --git a/tests/tests.targets b/tests/tests.targets index 8f00fd749956..6c3afcd7f134 100644 --- a/tests/tests.targets +++ b/tests/tests.targets @@ -1,8 +1,6 @@ - - <_SkipTestAssemblies Include="$(SkipTestAssemblies)" /> @@ -26,10 +24,6 @@ - - @@ -37,20 +31,43 @@ category=outerloop;category=failing - true + all - + + + $(CORE_ROOT)\xunit.console.dll + + -parallel $(ParallelRun) + $(XunitArgs) -html $(__TestRunHtmlLog) + $(XunitArgs) -xml $(__TestRunXmlLog) + $(XunitArgs) @(IncludeTraitsItems->'-trait %(Identity)', ' ') + $(XunitArgs) @(ExcludeTraitsItems->'-notrait %(Identity)', ' ') + + $(CORE_ROOT)\corerun + $(CORE_ROOT)\corerun.exe + + + + + <_TestAssembliesRelative Include="@(TestAssemblies -> Replace('$(BaseOutputPathWithConfig)', '.\'))" /> + + + + + + $(CorerunExecutable) $(XunitConsoleRunner) @(TestAssemblies->'%(Identity)', ' ') $(XunitArgs) + + + + diff --git a/tests/xunitwrapper.targets b/tests/xunitwrapper.targets deleted file mode 100644 index a0ff7f59fa2d..000000000000 --- a/tests/xunitwrapper.targets +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - - - - - - - - - <_ProjectReferenceFilenames Include="@(_ResolvedProjectReferencePaths->'%(FileName)%(Extension)')"> - %(Identity) - - - <_ReferencesFileNames Include="@(Reference->'%(FileName)%(Extension)')"> - %(Identity) - - - <_CopyLocalFileNames Include="@(CopyLocal->'%(FileName)%(Extension)')"> - %(Identity) - - - - <_ReferenceFileNamesToRemove Include="@(_ReferencesFileNames->'%(OriginalIdentity)')" Condition="'@(_ProjectReferenceFilenames)' == '@(_ReferencesFileNames)' and '%(Identity)' != ''"/> - <_CopyLocalFileNamesToRemove Include="@(_CopyLocalFileNames->'%(OriginalIdentity)')" Condition="'@(_ProjectReferenceFilenames)' == '@(_CopyLocalFileNames)' and '%(Identity)' != ''"/> - - - - - - - - - - - - - -