[wasm] Use source built emsdk packages for the runtime build (dotnet#…

…100266)

* Use the emsdk transport packages to build wasm instead of a cloned emsdk

Works for the most part out of the box, but fails on the part where we npm install after
the libraries build. The reason for the failure is our node transport package is incomplete. If
you replace it with a legit node, the build works fully.

* Pull in node transport package as opposed to the one packaged in emsdk. Streamline package deps

* Make paths friendly for windows

* Work in windows transport packages, copy python to its own directory, and adjust a bunch of paths

* Bump to latest version of node that contains icu

* Add windows deps and add DOTNET_EMSCRIPTEN_NODE_PATH because windows can't see npm without it

* Bump emscripten packages to the latest

* Fix typos in Version.Details.xml

* Switch out to the plain mariner container to ensure no EMSDK already exists

* Revert back to the browser-wasm docker image and bump to the latest

* Container type-o

* Fix condition where the node path isn't set properly

* Provision even when building offsets

* Fix goofy paths

* Update new template to use the latest browser-wasm container

* Don't put python.exe on the path for windows, but the folder instead

* So that was a bad idea. May have to have emsdk_env.cmd have two entries

* Fix offsets generation for browser

* Fix emsdk's python path

It is not versioned anymore as we use the content of our nugets

* Fix EMSDK_PATH on non-windows platforms

* Do not add link flags to compile flags

To avoid:

    C:\helix\work\correlation\build\wasm-shared\WasmApp.Common.targets(825,5): error : emcc: warning: linker setting ignored during compilation: 'EXPORT_ES6' [-Wunused-command-line-argument] [C:\helix\work\workitem\e\publish\ProxyProjectForAOTOnHelix.proj]
    C:\helix\work\correlation\build\wasm-shared\WasmApp.Common.targets(825,5): error : emcc: warning: linker setting ignored during compilation: 'EXPORT_EXCEPTION_HANDLING_HELPERS' [-Wunused-command-line-argument] [C:\helix\work\workitem\e\publish\ProxyProjectForAOTOnHelix.proj]

* Set the emsdk paths relative to the script location

* Escape few characters and fix the added script

* Fix .emscripten script

* Use EM_CONFIG intead of __file__

* Fix emsdk_env.cmd script

* Revert "Do not add link flags to compile flags"

This reverts commit eb19ade.

* Revert changes in package-lock.json

* Feedback + cleaning

* More cleaning

* Try not to use the replace in the EMSDK_PATH

* Better property names

* Use package and emsdk version properties

Co-authored-by: Larry Ewing <[email protected]>

* Use the updated properties

* Fix emsdk version

* Use a single version of emsdk and take node version from flow

* Update eng/Version.Details.xml

Co-authored-by: Larry Ewing <[email protected]>

---------

Co-authored-by: Alexander Köplinger <[email protected]>
Co-authored-by: Radek Doulik <[email protected]>
Co-authored-by: Radek Doulik <[email protected]>
Co-authored-by: Larry Ewing <[email protected]>

eng/Version.Details.xml

@@ -414,5 +414,37 @@
       <Uri>https://github.com/dotnet/installer</Uri>
       <Sha>fa261b952d702c6bd604728fcbdb58ac071a22b1</Sha>
     </Dependency>
+    <Dependency Name="runtime.linux-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.linux-musl-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.linux-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.linux-musl-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.osx-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.osx-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.win-arm64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
+    <Dependency Name="runtime.win-x64.Microsoft.NETCore.Runtime.Wasm.Node.Transport" Version="9.0.0-alpha.1.24175.1">
+      <Uri>https://github.com/dotnet/node</Uri>
+      <Sha>308c7d0f1fa19bd1e7b768ad13646f5206133cdb</Sha>
+    </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -256,5 +256,9 @@
     <!-- installer version, for testing workloads -->
     <MicrosoftDotnetSdkInternalVersion>9.0.100-preview.5.24253.16</MicrosoftDotnetSdkInternalVersion>
     <SdkVersionForWorkloadTesting>$(MicrosoftDotnetSdkInternalVersion)</SdkVersionForWorkloadTesting>
+    <runtimewinx64MicrosoftNETCoreRuntimeWasmNodeTransportPackageVersion>9.0.0-alpha.1.24175.1</runtimewinx64MicrosoftNETCoreRuntimeWasmNodeTransportPackageVersion>
+    <EmsdkPackageVersion>$(MicrosoftNETRuntimeEmscriptenVersion)</EmsdkPackageVersion>
+    <NodePackageVersion>$(runtimewinx64MicrosoftNETCoreRuntimeWasmNodeTransportPackageVersion)</NodePackageVersion>
+    <EmsdkVersion>3.1.34</EmsdkVersion>
   </PropertyGroup>
 </Project>

eng/native/gen-buildsys.cmd

@@ -56,7 +56,7 @@ if /i "%__Arch%" == "wasm" (
         set "EMSDK_PATH=!EMSDK_PATH:\=/!"
         if not "!EMSDK_PATH:~-1!" == "/" set "EMSDK_PATH=!EMSDK_PATH!/"
 
-        set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_TOOLCHAIN_FILE=!EMSDK_PATH!/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake"
+        set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_TOOLCHAIN_FILE=!EMSDK_PATH!/emscripten/cmake/Modules/Platform/Emscripten.cmake"
         set __UseEmcmake=1
     )
     if /i "%__Os%" == "wasi" (
@@ -107,7 +107,7 @@ if not "%__ConfigureOnly%" == "1" (
 )
 
 if /i "%__UseEmcmake%" == "1" (
-    call "!EMSDK_PATH!/emsdk_env.bat" > nul 2>&1 && emcmake "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%
+    call "!EMSDK_PATH!/emsdk_env.cmd" > nul 2>&1 && emcmake "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%
 ) else (
     "%CMakePath%" %__ExtraCmakeParams% --no-warn-unused-cli -G "%__CmakeGenerator%" -B %__IntermediatesDir% -S %__SourceDir%
 )

eng/pipelines/common/templates/pipeline-with-resources.yml

@@ -100,7 +100,7 @@ extends:
         image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8
 
       browser_wasm:
-        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly
         env:
           ROOTFS_DIR: /crossrootfs/x64
 

src/libraries/sendtohelix-browser.targets

@@ -52,8 +52,7 @@
     <IncludeHelixCorrelationPayload>false</IncludeHelixCorrelationPayload>
     <EnableDefaultBuildHelixWorkItems>false</EnableDefaultBuildHelixWorkItems>
 
-    <!-- on unix CI has emscripten provisioned in $(EMSDK_PATH) as `/usr/local/emscripten`. -->
-    <EMSDK_PATH Condition="$([MSBuild]::IsOSPlatform('WINDOWS')) and '$(EMSDK_PATH)' == ''">$(RepoRoot)src\mono\browser\emsdk\</EMSDK_PATH>
+    <EMSDK_PATH Condition="'$(EMSDK_PATH)' == ''">$([MSBuild]::NormalizeDirectory($(RepoRoot), 'src', 'mono', 'browser', 'emsdk'))</EMSDK_PATH>
     <EmSdkDirForHelixPayload>$(HelixDependenciesStagingPath)\emsdk</EmSdkDirForHelixPayload>
     <DebuggerHost Condition="'$(DebuggerHost)' == ''">chrome</DebuggerHost>
 

src/mono/browser/Makefile

@@ -167,4 +167,4 @@ build-app-host:
 	$(DOTNET) build $(TOP)/src/mono/wasm/host $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS)
 
 patch-deterministic:
-	cd emsdk/upstream/emscripten/ && patch -p1 < ../../../runtime/deterministic.diff
+	cd emsdk/emscripten/ && patch -p1 < ../../../runtime/deterministic.diff

src/mono/browser/browser.proj

@@ -33,7 +33,7 @@
     <_EmccDefaultsRspPath>$(NativeBinDir)src\emcc-default.rsp</_EmccDefaultsRspPath>
     <_EmccCompileRspPath>$(NativeBinDir)src\emcc-compile.rsp</_EmccCompileRspPath>
     <_EmccLinkRspPath>$(NativeBinDir)src\emcc-link.rsp</_EmccLinkRspPath>
-    <EmSdkLLVMAr>$(EMSDK_PATH)\upstream\bin\llvm-ar</EmSdkLLVMAr>
+    <EmSdkLLVMAr>$(EMSDK_PATH)\bin\llvm-ar</EmSdkLLVMAr>
     <EmSdkLLVMAr Condition="$([MSBuild]::IsOSPlatform('Windows'))">$(EmSdkLLVMAr).exe</EmSdkLLVMAr>
   </PropertyGroup>
 
@@ -397,12 +397,12 @@
       <CMakeBuildRuntimeConfigureCmd Condition="'$(WasmEnableJsInteropByValue)' == 'true'">$(CMakeBuildRuntimeConfigureCmd) -DENABLE_JS_INTEROP_BY_VALUE=1</CMakeBuildRuntimeConfigureCmd>
       <CMakeBuildRuntimeConfigureCmd>$(CMakeBuildRuntimeConfigureCmd) $(CMakeConfigurationEmsdkPath)</CMakeBuildRuntimeConfigureCmd>
 
-      <CMakeBuildRuntimeConfigureCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.bat'))&quot; &amp;&amp; $(CMakeBuildRuntimeConfigureCmd)</CMakeBuildRuntimeConfigureCmd>
+      <CMakeBuildRuntimeConfigureCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.cmd'))&quot; &amp;&amp; $(CMakeBuildRuntimeConfigureCmd)</CMakeBuildRuntimeConfigureCmd>
       <CMakeBuildRuntimeConfigureCmd Condition="'$(OS)' != 'Windows_NT'">bash -c 'source $(EMSDK_PATH)/emsdk_env.sh 2>&amp;1 &amp;&amp; $(CMakeBuildRuntimeConfigureCmd)'</CMakeBuildRuntimeConfigureCmd>
 
       <CMakeOptions Condition="'$(MonoVerboseBuild)' != ''">-v</CMakeOptions>
       <CMakeBuildRuntimeCmd>cmake --build . --config $(Configuration) $(CmakeOptions)</CMakeBuildRuntimeCmd>
-      <CMakeBuildRuntimeCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.bat'))&quot; &amp;&amp; $(CMakeBuildRuntimeCmd)</CMakeBuildRuntimeCmd>
+      <CMakeBuildRuntimeCmd Condition="'$(OS)' == 'Windows_NT'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; &amp;&amp; call &quot;$([MSBuild]::NormalizePath('$(EMSDK_PATH)', 'emsdk_env.cmd'))&quot; &amp;&amp; $(CMakeBuildRuntimeCmd)</CMakeBuildRuntimeCmd>
       <CMakeBuildRuntimeCmd Condition="'$(OS)' != 'Windows_NT'">bash -c 'source $(EMSDK_PATH)/emsdk_env.sh 2>&amp;1 &amp;&amp;  $(CMakeBuildRuntimeCmd)'</CMakeBuildRuntimeCmd>
     </PropertyGroup>
     <ItemGroup>

src/mono/browser/build/EmSdkRepo.Defaults.props

@@ -4,25 +4,18 @@
 
   <PropertyGroup>
     <EmscriptenSdkToolsPath>$([MSBuild]::EnsureTrailingSlash($(EMSDK_PATH)))</EmscriptenSdkToolsPath>
-    <EmscriptenUpstreamBinPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'upstream', 'bin'))</EmscriptenUpstreamBinPath>
-    <EmscriptenUpstreamEmscriptenPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'upstream', 'emscripten'))</EmscriptenUpstreamEmscriptenPath>
-
-    <_NodeToolsBasePath>$(EmscriptenSdkToolsPath)node</_NodeToolsBasePath>
+    <EmscriptenUpstreamBinPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'bin'))</EmscriptenUpstreamBinPath>
+    <EmscriptenUpstreamEmscriptenPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'emscripten'))</EmscriptenUpstreamEmscriptenPath>
 
     <!-- gets the path like emsdk/python/3.7.4-2_64bit -->
-    <_NodeToolsVersionedPath Condition="Exists($(_NodeToolsBasePath))">$([System.IO.Directory]::GetDirectories($(_NodeToolsBasePath)))</_NodeToolsVersionedPath>
-    <EmscriptenNodeToolsPath Condition="'$(_NodeToolsVersionedPath)' != ''">$(_NodeToolsVersionedPath)</EmscriptenNodeToolsPath>
-    <EmscriptenNodeToolsPath Condition="'$(EmscriptenNodeToolsPath)' != ''">$([MSBuild]::NormalizeDirectory($(EmscriptenNodeToolsPath)))</EmscriptenNodeToolsPath>
+    <EmscriptenNodeToolsPath Condition="'$(EmscriptenNodeToolsPath)' == ''">$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'node', 'bin'))</EmscriptenNodeToolsPath>
 
     <_UsingEMSDK_PATH>true</_UsingEMSDK_PATH>
   </PropertyGroup>
 
   <!-- Emscripten uses system python on Linux, so we don't need $(EmscriptenPythonToolsPath) -->
   <PropertyGroup Condition="!$([MSBuild]::IsOSPlatform('linux'))">
-    <_PythonToolsBasePath>$(EmscriptenSdkToolsPath)python</_PythonToolsBasePath>
-    <_PythonToolsVersionedPath Condition="Exists($(_PythonToolsBasePath))">$([System.IO.Directory]::GetDirectories($(_PythonToolsBasePath)))</_PythonToolsVersionedPath>
-    <EmscriptenPythonToolsPath Condition="'$(_PythonToolsVersionedPath)' != ''">$(_PythonToolsVersionedPath)</EmscriptenPythonToolsPath>
-    <EmscriptenPythonToolsPath Condition="'$(EmscriptenPythonToolsPath)' != ''">$([MSBuild]::NormalizeDirectory($(EmscriptenPythonToolsPath)))</EmscriptenPythonToolsPath>
+    <EmscriptenPythonToolsPath>$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath)python))</EmscriptenPythonToolsPath>
 
     <_EMSDKMissingPaths Condition="'$(_EMSDKMissingPaths)' == '' and ('$(EmscriptenPythonToolsPath)' == '' or !Exists('$(EmscriptenPythonToolsPath)'))">%24(EmscriptenPythonToolsPath)=$(EmscriptenPythonToolsPath) </_EMSDKMissingPaths>
   </PropertyGroup>
@@ -31,9 +24,9 @@
   <ItemGroup>
     <EmscriptenPrependPATH Include="$(EmscriptenSdkToolsPath)" />
     <EmscriptenPrependPATH Include="$(EmscriptenUpstreamBinPath)" />
-    <EmscriptenPrependPATH Include="$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'upstream', 'emscripten'))" />
+    <EmscriptenPrependPATH Include="$([MSBuild]::NormalizeDirectory($(EmscriptenSdkToolsPath), 'emscripten'))" />
 
-    <EmscriptenPrependPATH Include="$(EmscriptenNodeToolsPath)bin" />
+    <EmscriptenPrependPATH Include="$(EmscriptenNodeToolsPath)" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(EmscriptenPythonToolsPath)' != ''">

src/mono/browser/runtime/CMakeLists.txt

@@ -44,7 +44,7 @@ set(ignoreMeEmsdkPath "${EMSDK_PATH}")
 
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
     add_custom_command(TARGET dotnet.native
-                        POST_BUILD COMMAND ${EMSDK_PATH}/upstream/bin/wasm-opt --enable-exception-handling ${CONFIGURATION_WASM_OPT_FLAGS} --strip-dwarf ${NATIVE_BIN_DIR}/dotnet.native.wasm -o ${NATIVE_BIN_DIR}/dotnet.native.wasm
+                        POST_BUILD COMMAND ${EMSDK_PATH}/bin/wasm-opt --enable-exception-handling ${CONFIGURATION_WASM_OPT_FLAGS} --strip-dwarf ${NATIVE_BIN_DIR}/dotnet.native.wasm -o ${NATIVE_BIN_DIR}/dotnet.native.wasm
                         COMMENT "Stripping debug symbols from dotnet.native.wasm using wasm-opt")
 endif()
 

src/mono/browser/sanitize.py

@@ -32,7 +32,7 @@ def rewrite_package_json(path):
 
 
 emsdk_path = sys.argv[1]
-emscripten_path = os.path.join(emsdk_path, "upstream", "emscripten")
+emscripten_path = os.path.join(emsdk_path, "emscripten")
 node_root = os.path.join(emsdk_path, "node")
 node_paths = glob(node_root)
 upgrade = True