Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[libraries] Skip test suites crashing on CI for iOS/tvOS/MacCatalyst #57208

Merged
merged 38 commits into from
Sep 3, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
08eb601
[MacCatalyst][Libraries] Reenable MacCatalyst x64 crashing test suites
Aug 11, 2021
8490e6d
[MacCatalyst][Libraries] Reenable MacCatalyst arm64 crashing test suites
Aug 11, 2021
9f8629e
Temporarily enable MacCatalyst arm64 lane
Aug 11, 2021
139e967
Readd consistently crashing suites to iOS tvOS MacCatalyst skips
Aug 16, 2021
f98dfa7
Address crashes and PNSE failures on iOS MacCatalyst
Aug 16, 2021
3326f3d
Temporarily bump timeout for tvOS
Aug 16, 2021
acc8104
fix typo
Aug 16, 2021
f62cc55
Skip more crashed suites
Aug 17, 2021
f12648d
Undo temporary changes and Add ActiveIssue
Aug 17, 2021
8ef4da1
Skip more crashing suites
Aug 17, 2021
29c7d44
Skip iOS.Simulator.Aot.Test flake on MacCatalyst CI
Aug 17, 2021
9e65ef0
Add a couple crashing suites
Aug 18, 2021
d975c37
Remove sample skip in tests.proj as it doesnt do anything
Aug 19, 2021
f8854ff
Add another flakey crashed test for MacCatalyst
Aug 19, 2021
1db446d
Skip System.Xml.Xsl.XslCompiledTransformApi on MacCatalyst
Aug 25, 2021
7b8a614
Temporarily remove test suite skips and enable arm64 lanes and bump l…
Aug 26, 2021
70ef7ec
Reenable a test that was fixed
akoeplinger Aug 27, 2021
3976d6d
Merge branch 'main' into macCatalyst_run_crashing_tests
akoeplinger Aug 27, 2021
83739a1
Update Microsoft.DotNet.Helix.Sdk
akoeplinger Aug 27, 2021
5ee3b26
[iOS] Skip ValidAccessLevelCombinations ReadWriteExecute for iOSSimul…
Aug 30, 2021
018fa0d
Skip failing test suites
Aug 30, 2021
98ee28a
Extend trampoline bump to iOS tvOS MacCatalyst
Aug 30, 2021
b70a72e
Fixup typo
Aug 30, 2021
04d680f
Fix MonoAOTCompiler typo
Aug 30, 2021
2c1fcc5
Bump trampolines for osxlike lanes
Aug 31, 2021
a8b9bf4
Add active issues for failing tests
Aug 31, 2021
54b711b
Merge remote-tracking branch 'upstream/main' into macCatalyst_run_cra…
Aug 31, 2021
65d9368
Remove ActiveIssue that was fixed
akoeplinger Sep 1, 2021
b083ec9
Add IsIanaIdTest skip
Sep 1, 2021
a9d1b04
Bump trampolines even higher
Sep 1, 2021
7cf1101
Address feedback
Sep 1, 2021
8f83490
Fix typo
Sep 1, 2021
98698fd
Undo runtime-staging temporary changes and cleanup test skips
Sep 1, 2021
4317e54
Fix path typo
Sep 1, 2021
c5668c5
Skip iOS sample
Sep 2, 2021
8d4db8d
Add new activeIssue and skip flakey Android test crash
Sep 2, 2021
8365cdd
Skip functional tests crashing with exit code 0
Sep 3, 2021
ec9a307
Merge remote-tracking branch 'upstream/main' into macCatalyst_run_cra…
Sep 3, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public void PollingFileProviderShouldntConsumeINotifyInstances()
}
}
}

private void GetFileInfoReturnsNotFoundFileInfoForIllegalPathWithLeadingSlashes(string path)
{
using (var provider = new PhysicalFileProvider(Path.GetTempPath()))
Expand Down Expand Up @@ -936,6 +936,7 @@ public void NoopChangeTokenForFilterThatNavigatesAboveRoot()

[Fact]
[ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/58584", TestPlatforms.iOS | TestPlatforms.MacCatalyst | TestPlatforms.tvOS)]
public void TokenForEmptyFilter()
{
using (var root = new DisposableFileSystem())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ public void NonZeroTimeout_NoEvents_TimesOut(bool enabledBeforeWait)
[InlineData(WatcherChangeTypes.Changed, false)]
[InlineData(WatcherChangeTypes.Renamed, true)]
[InlineData(WatcherChangeTypes.All, true)]
[ActiveIssue("https://github.com/dotnet/runtime/issues/58418", typeof(PlatformDetection), nameof(PlatformDetection.IsMacCatalyst), nameof(PlatformDetection.IsArm64Process))]
public void NonZeroTimeout_NoActivity_TimesOut(WatcherChangeTypes changeType, bool enabledBeforeWait)
{
using (var testDirectory = new TempDirectory(GetTestFilePath()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ public void ValidAccessLevelCombinations(MemoryMappedFileAccess mapAccess, Memor
}
catch (UnauthorizedAccessException)
{
if ((OperatingSystem.IsMacOS() || OperatingSystem.IsMacCatalyst() || PlatformDetection.IsInContainer) &&
if ((OperatingSystem.IsMacOS() || OperatingSystem.IsMacCatalyst() || OperatingSystem.IsIOS() || OperatingSystem.IsTvOS() || PlatformDetection.IsInContainer) &&
(viewAccess == MemoryMappedFileAccess.ReadExecute || viewAccess == MemoryMappedFileAccess.ReadWriteExecute))
{
// Containers and OSX with SIP enabled do not have execute permissions by default.
// Containers and OSXlike platforms with SIP enabled do not have execute permissions by default.
throw new SkipTestException("Insufficient execute permission.");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ public void ValidAccessLevelCombinations(MemoryMappedFileAccess mapAccess, Memor
}
catch (UnauthorizedAccessException)
{
if ((OperatingSystem.IsMacOS() || OperatingSystem.IsMacCatalyst() || PlatformDetection.IsInContainer) &&
if ((OperatingSystem.IsMacOS() || OperatingSystem.IsMacCatalyst() || OperatingSystem.IsIOS() || OperatingSystem.IsTvOS() || PlatformDetection.IsInContainer) &&
(viewAccess == MemoryMappedFileAccess.ReadExecute || viewAccess == MemoryMappedFileAccess.ReadWriteExecute))
{
// Containers and OSX with SIP enabled do not have execute permissions by default.
// Containers and OSXlike platforms with SIP enabled do not have execute permissions by default.
throw new SkipTestException("Insufficient execute permission.");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -346,11 +346,8 @@ public void GetFolderPath_Unix_PersonalIsHomeAndUserProfile()
Assert.Equal(Environment.GetEnvironmentVariable("HOME"), Environment.GetFolderPath(Environment.SpecialFolder.Personal));
Assert.Equal(Environment.GetEnvironmentVariable("HOME"), Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments));
}
// tvOS effectively doesn't have a HOME
if (!PlatformDetection.IsiOS && !PlatformDetection.IstvOS)
{
Assert.Equal(Environment.GetEnvironmentVariable("HOME"), Environment.GetFolderPath(Environment.SpecialFolder.UserProfile));
}

Assert.Equal(Environment.GetEnvironmentVariable("HOME"), Environment.GetFolderPath(Environment.SpecialFolder.UserProfile));
}

[Theory]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2660,6 +2660,7 @@ public static void IsIanaIdWithNotCacheTest()
}

[ConditionalFact(nameof(SupportIanaNamesConversion))]
[ActiveIssue("https://github.com/dotnet/runtime/issues/58440", TestPlatforms.iOS | TestPlatforms.MacCatalyst | TestPlatforms.tvOS)]
public static void IsIanaIdTest()
{
bool expected = !s_isWindows;
Expand Down
71 changes: 16 additions & 55 deletions src/libraries/tests.proj
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Private.Xml/tests/XPath/XPathDocument/System.Xml.XPath.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Private.Uri/tests/ExtendedFunctionalTests/System.Private.Uri.ExtendedFunctional.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Reflection.DispatchProxy/tests/System.Reflection.DispatchProxy.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Security.Cryptography.X509Certificates\tests\System.Security.Cryptography.X509Certificates.Tests.csproj" />
</ItemGroup>

<ItemGroup Condition="'$(TargetOS)' == 'Android' and '$(TargetArchitecture)' == 'x64' and '$(RunDisabledAndroidTests)' != 'true'">
Expand All @@ -145,68 +146,27 @@
<!-- https://github.com/dotnet/runtime/issues/51414 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Security.Cryptography.OpenSsl/tests/System.Security.Cryptography.OpenSsl.Tests.csproj" />

<!-- Failures -->
<!-- https://github.com/dotnet/runtime/issues/51335 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Pipes/tests/System.IO.Pipes.Tests.csproj" />

<!-- Crashes or hangs, did not produce testResults.xml -->
<!-- https://github.com/dotnet/runtime/issues/52120 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Net.Security/tests/FunctionalTests/System.Net.Security.Tests.csproj" />

<!-- Crashes on CI but passes locally https://github.com/dotnet/runtime/issues/52615 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.Extensions.Http/tests/Microsoft.Extensions.Http.Tests/Microsoft.Extensions.Http.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.Extensions.Logging/tests/Common/Microsoft.Extensions.Logging.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Compression/tests/System.IO.Compression.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Compression.Brotli/tests/System.IO.Compression.Brotli.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Private.Uri/tests/UnitTests/System.Private.Uri.Unit.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Private.Xml/tests/XmlResolver/System.Xml.XmlResolver.Tests/System.Xml.XmlResolver.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Security.Cryptography.Csp/tests/System.Security.Cryptography.Csp.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Threading.Tasks.Dataflow/tests/System.Threading.Tasks.Dataflow.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Threading.Tasks/tests/System.Threading.Tasks.Tests.csproj" />

<!-- App Crash https://github.com/dotnet/runtime/issues/53624 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.CSharp/tests/Microsoft.CSharp.Tests.csproj" />
</ItemGroup>

<ItemGroup Condition="('$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'tvOSSimulator') and '$(RunDisablediOSTests)' != 'true'">
<!-- https://github.com/dotnet/runtime/issues/50589 -->
<ProjectExclusions Include="$(RepoRoot)\src\tests\FunctionalTests\tvOS\Simulator\AOT\tvOS.Simulator.Aot.Test.csproj" />

<!-- Crashes on CI but passes locally https://github.com/dotnet/runtime/issues/52615 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Compression.Brotli/tests/System.IO.Compression.Brotli.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Compression.ZipFile/tests/System.IO.Compression.ZipFile.Tests.csproj" />
</ItemGroup>

<ItemGroup Condition="'$(TargetOS)' == 'MacCatalyst' and '$(RunDisablediOSTests)' != 'true'">
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Diagnostics.Tracing/tests/System.Diagnostics.Tracing.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Dynamic.Runtime/tests/System.Dynamic.Runtime.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Net.Requests/tests/System.Net.Requests.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Net.Security/tests/FunctionalTests/System.Net.Security.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj" />
<ProjectExclusions Include="$(RepoRoot)/src/tests/FunctionalTests/iOS/Simulator/PInvoke/iOS.Simulator.PInvoke.Test.csproj" />
<ProjectExclusions Include="$(RepoRoot)/src/tests/FunctionalTests/tvOS/Simulator/AOT/tvOS.Simulator.Aot.Test.csproj" />
<!-- Crashes randomly during test runs https://github.com/dotnet/runtime/issues/52460 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Threading.Tasks\tests\System.Threading.Tasks.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Threading.Tasks.Dataflow\tests\System.Threading.Tasks.Dataflow.Tests.csproj" />

<!-- Crash https://github.com/dotnet/runtime/issues/56085 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.Json\tests\System.Text.Json.SourceGeneration.Tests\System.Text.Json.SourceGeneration.Tests.csproj" />

<!-- Crashes on CI but passes locally https://github.com/dotnet/runtime/issues/52615 -->
<ProjectExclusions Include="$(RepoRoot)\src\tests\FunctionalTests\iOS\Simulator\Interpreter\iOS.Simulator.Interpreter.Test.csproj" />
<ProjectExclusions Include="$(RepoRoot)\src\tests\FunctionalTests\iOS\Simulator\InvariantCultureOnlyMode\iOS.Simulator.InvariantCultureOnlyMode.Test.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Tests.csproj" />
</ItemGroup>

<ItemGroup Condition="'$(TargetOS)' == 'MacCatalyst' and '$(TargetArchitecture)' == 'arm64' and '$(RunDisablediOSTests)' != 'true'">
<!-- App Crash https://github.com/dotnet/runtime/issues/53624 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.FileSystem.Watcher/tests/System.IO.FileSystem.Watcher.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj" />
</ItemGroup>

<ItemGroup Condition="'$(TargetOS)' == 'MacCatalyst' and '$(TargetArchitecture)' == 'x64' and '$(RunDisablediOSTests)' != 'true'">
<!-- App Crash https://github.com/dotnet/runtime/issues/53624 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.CSharp/tests/Microsoft.CSharp.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Diagnostics.Tracing/tests/System.Diagnostics.Tracing.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Dynamic.Runtime/tests/System.Dynamic.Runtime.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Linq.Parallel/tests/System.Linq.Parallel.Tests.csproj" />
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj" />
<ItemGroup Condition="('$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'tvOSSimulator') and '$(RunDisablediOSTests)' != 'true'">
<!-- https://github.com/dotnet/runtime/issues/51335 -->
<ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.Pipes/tests/System.IO.Pipes.Tests.csproj" />
</ItemGroup>

<ItemGroup Condition="'$(TargetOS)' == 'Browser' and '$(RunDisabledWasmTests)' != 'true'">
Expand Down Expand Up @@ -303,8 +263,9 @@
</ItemGroup>

<ItemGroup Condition="'$(ArchiveTests)' == 'true' and ('$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'MacCatalyst')">
<ProjectReference Include="$(MonoProjectRoot)sample\iOS\Program.csproj"
BuildInParallel="false" />
<!-- https://github.com/dotnet/runtime/issues/57666 -->
<!-- <ProjectReference Include="$(MonoProjectRoot)sample\iOS\Program.csproj"
BuildInParallel="false" /> -->

<ProjectReference Include="$(RepoRoot)\src\tests\FunctionalTests\iOS\**\*.Test.csproj"
Exclude="@(ProjectExclusions)"
Expand Down
3 changes: 2 additions & 1 deletion src/tasks/AotCompilerTask/MonoAOTCompiler.props
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
<MonoAOTCompilerDefaultAotArguments Condition="'$(TargetArchitecture)' == 'arm64'" Include="mtriple=aarch64-linux-android" />
<MonoAOTCompilerDefaultAotArguments Condition="'$(TargetArchitecture)' == 'x86'" Include="mtriple=i686-linux-android" />
<MonoAOTCompilerDefaultAotArguments Condition="'$(TargetArchitecture)' == 'x64'" Include="mtriple=x86_64-linux-android" />

</ItemGroup>
<ItemGroup Condition="'$(TargetOS)' == 'Android' or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'tvOSSimulator' or '$(TargetOS)' == 'MacCatalyst'">
<!-- Default trampolines run out for libraries tests -->
<MonoAOTCompilerDefaultAotArguments Include="nimt-trampolines=2000" />
<MonoAOTCompilerDefaultAotArguments Include="ntrampolines=10000" />
Expand Down