Use babel for transpiling rather than closure compiler (#20879)

We recently switch from using closure-compiler with `WHITESPACE_ONLY` to
closure-compiler with `SIMPLE_OPTIMIZATIONS`.  However this had the
negative side effect that all input need to be free of closure compiler
warnings, and this turned out not to be practical for all users.
See #20810 for more on this

When selecting a transpilation tool use I also evaluated `swx` (written
in rust) and `esbuild` (written in go).  `esbuild` was rejected
because the simply don't support ES5
(evanw/esbuild#297).  `swx` was rejected
because it almost doubled the side of our `node_modules` directory by
adding two 50mb binary files.

.flake8

@@ -1,6 +1,7 @@
 [flake8]
 ignore = E111,E114,E501,E261,E266,E121,E402,E241,W504,E741,B011,B023,U101
 exclude =
+ ./node_modules/, # third-party code
  ./third_party/, # third-party code
  ./tools/filelock.py, # third-party code
  ./tools/scons/, # third-party code

ChangeLog.md

@@ -29,15 +29,20 @@ See docs/process.md for more on how version tagging works.
   ports of native GL renderers from later accidentally attempting to activate
   "dormant" features if web browser implementations gain new WebGL extensions in
   the future, which `*glGetProcAddress()` is not able to support. (#20802)
-- Added Hi DPI support to GLFW. When enabled, GLFW automatically accounts for the
-  `devicePixelRatio` browser property and changes the size of the canvas accordingly
-  (including dynamically if the canvas is moved from a 4k screen to a 2k screen and 
-  vice-versa). `glfwGetFramebufferSize` now properly returns the canvas size in pixels, 
-  while `glfwGetWindowSize` returns the canvas size is screen size. By default,
-  this feature is disabled. You can enable it before creating a window by calling 
-  `glfwWindowHint(GLFW_SCALE_TO_MONITOR, GLFW_TRUE)`. You can also 
-  dynamically change it after the window has been created by calling
-  `glfwSetWindowAttrib(window, GLFW_SCALE_TO_MONITOR, GLFW_TRUE)`. (#20584)
+- Added Hi DPI support to GLFW. When enabled, GLFW automatically accounts for
+  the `devicePixelRatio` browser property and changes the size of the canvas
+  accordingly (including dynamically if the canvas is moved from a 4K screen to
+  a 2K screen and vice-versa). `glfwGetFramebufferSize` now properly returns the
+  canvas size in pixels, while `glfwGetWindowSize` returns the canvas size is
+  screen size. By default, this feature is disabled. You can enable it before
+  creating a window by calling `glfwWindowHint(GLFW_SCALE_TO_MONITOR,
+  GLFW_TRUE)`. You can also dynamically change it after the window has been
+  created by calling `glfwSetWindowAttrib(window, GLFW_SCALE_TO_MONITOR,
+  GLFW_TRUE)`. (#20584)
+- Transpilation to support older environments/browsers is now performed by babel
+  rather than closure compiler.  This means that folks targeting older browsers
+  (e.g. `-sLEGACY_VM_SUPPORT`) do not need to ensure their code is closure
+  compliant. (#20879)
 
 3.1.50 - 11/29/23
 -----------------

package.json

@@ -11,6 +11,9 @@
     "ws": "^8.14.2"
   },
   "dependencies": {
+    "@babel/cli": "^7.23.4",
+    "@babel/core": "^7.23.5",
+    "@babel/preset-env": "^7.23.5",
     "acorn": "^8.11.2",
     "google-closure-compiler": "20230802.0.0",
     "html-minifier-terser": "7.2.0"

src/settings_internal.js

@@ -243,10 +243,9 @@ var LINK_AS_CXX = false;
 // emitted in that case for closure compiler.
 var MAYBE_CLOSURE_COMPILER = false;
 
-// Set when some minimum browser version triggers doesn't support the
-// minimum set of ES6 features.  This triggers transpilation to ES5
-// using closure compiler.
-var TRANSPILE_TO_ES5 = false;
+// Set when some minimum browser version triggers doesn't support the minimum
+// set of JavaScript features.  This triggers transpilation using babel.
+var TRANSPILE = false;
 
 // A copy of the default the default INCOMING_MODULE_JS_API. (Soon to
 // include additional items).

test/test_other.py

@@ -13130,7 +13130,6 @@ def check_for_es6(filename, expect):
         self.assertContained('foo(arg="hello")', js)
         self.assertContained(['() => 2', '()=>2'], js)
         self.assertContained('const ', js)
-        self.assertContained('let ', js)
         self.assertContained('?.[', js)
         self.assertContained('?.(', js)
         self.assertContained('??=', js)
@@ -13142,7 +13141,6 @@ def check_for_es6(filename, expect):
         self.assertNotContained('() => 2', js)
         self.assertNotContained('()=>2', js)
         self.assertNotContained('const ', js)
-        self.assertNotContained('let ', js)
         self.assertNotContained('??', js)
         self.assertNotContained('?.', js)
         self.assertNotContained('||=', js)
@@ -13162,13 +13160,12 @@ def check_for_es6(filename, expect):
     self.do_runf('test.c', expected, output_basename='test_old')
     check_for_es6('test_old.js', False)
 
-    # If we add `--closure=0` that transpiler (closure) is not run at all
-    print('with old browser + --closure=0')
-    self.do_runf('test.c', expected, emcc_args=['--closure=0'], output_basename='test_no_closure')
+    # If we add `-sPOLYFILL=0` that transpiler is not run at all
+    print('with old browser + -sPOLYFILL=0')
+    self.do_runf('test.c', expected, emcc_args=['-sPOLYFILL=0'], output_basename='test_no_closure')
     check_for_es6('test_no_closure.js', True)
 
-    # If we use `--closure=1` closure will run in full optimization mode
-    # and also transpile to ES5
+    # Test that transpiling is compatible with `--closure=1`
     print('with old browser + --closure=1')
     self.do_runf('test.c', expected, emcc_args=['--closure=1'], output_basename='test_closure')
     check_for_es6('test_closure.js', False)

tools/building.py

@@ -33,6 +33,7 @@
 from .shared import get_emscripten_temp_dir, exe_suffix, is_c_symbol
 from .utils import WINDOWS
 from .settings import settings, default_setting
+from .feature_matrix import UNSUPPORTED
 
 logger = logging.getLogger('building')
 
@@ -509,14 +510,39 @@ def add_to_path(dirname):
   return closure_cmd, env
 
 
+def version_split(v):
+  """Split version setting number (e.g. 162000) into versions string (e.g. "16.2.0")
+  """
+  v = str(v).rjust(6, '0')
+  assert len(v) == 6
+  m = re.match(r'(\d{2})(\d{2})(\d{2})', v)
+  major, minor, rev = m.group(1, 2, 3)
+  return f'{int(major)}.{int(minor)}.{int(rev)}'
+
+
 @ToolchainProfiler.profile()
-def closure_transpile(filename):
-  user_args = []
-  closure_cmd, env = get_closure_compiler_and_env(user_args)
-  closure_cmd += ['--language_out', 'ES5']
-  closure_cmd += ['--compilation_level', 'SIMPLE_OPTIMIZATIONS']
-  closure_cmd += ['--formatting', 'PRETTY_PRINT']
-  return run_closure_cmd(closure_cmd, filename, env)
+def transpile(filename):
+  config = {
+    'sourceType': 'script',
+    'targets': {}
+  }
+  if settings.MIN_CHROME_VERSION != UNSUPPORTED:
+    config['targets']['chrome'] = str(settings.MIN_CHROME_VERSION)
+  if settings.MIN_FIREFOX_VERSION != UNSUPPORTED:
+    config['targets']['firefox'] = str(settings.MIN_FIREFOX_VERSION)
+  if settings.MIN_IE_VERSION != UNSUPPORTED:
+    config['targets']['ie'] = str(settings.MIN_IE_VERSION)
+  if settings.MIN_SAFARI_VERSION != UNSUPPORTED:
+    config['targets']['safari'] = version_split(settings.MIN_SAFARI_VERSION)
+  if settings.MIN_NODE_VERSION != UNSUPPORTED:
+    config['targets']['node'] = version_split(settings.MIN_NODE_VERSION)
+  config_json = json.dumps(config, indent=2)
+  outfile = shared.get_temp_files().get('babel.js').name
+  config_file = shared.get_temp_files().get('babel_config.json').name
+  utils.write_file(config_file, config_json)
+  cmd = shared.get_npm_cmd('babel') + [filename, '-o', outfile, '--presets', '@babel/preset-env', '--config-file', config_file]
+  check_call(cmd, cwd=path_from_root())
+  return outfile
 
 
 @ToolchainProfiler.profile()
@@ -581,13 +607,8 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None):
   args = ['--compilation_level', 'ADVANCED_OPTIMIZATIONS' if advanced else 'SIMPLE_OPTIMIZATIONS']
   # Keep in sync with ecmaVersion in tools/acorn-optimizer.mjs
   args += ['--language_in', 'ECMASCRIPT_2021']
-  # Tell closure not to do any transpiling or inject any polyfills.
-  # At some point we may want to look into using this as way to convert to ES5 but
-  # babel is perhaps a better tool for that.
-  if settings.TRANSPILE_TO_ES5:
-    args += ['--language_out', 'ES5']
-  else:
-    args += ['--language_out', 'NO_TRANSPILE']
+  # We do transpilation using babel
+  args += ['--language_out', 'NO_TRANSPILE']
   # Tell closure never to inject the 'use strict' directive.
   args += ['--emit_use_strict=false']
 

tools/feature_matrix.py

@@ -13,6 +13,8 @@
 
 logger = logging.getLogger('feature_matrix')
 
+UNSUPPORTED = 0x7FFFFFFF
+
 
 class Feature(IntEnum):
   NON_TRAPPING_FPTOINT = auto()
@@ -91,7 +93,7 @@ def report_missing(setting_name):
     report_missing('MIN_SAFARI_VERSION')
     return False
   # IE don't support any non-MVP features
-  if settings.MIN_IE_VERSION != 0x7FFFFFFF:
+  if settings.MIN_IE_VERSION != UNSUPPORTED:
     report_missing('MIN_IE_VERSION')
     return False
   if 'node' in min_versions and settings.MIN_NODE_VERSION < min_versions['node']:

tools/link.py

@@ -1125,7 +1125,7 @@ def phase_linker_setup(options, state, newargs):
 
   setup_environment_settings()
 
-  if options.use_closure_compiler != 0 and settings.POLYFILL:
+  if settings.POLYFILL:
     # Emscripten requires certain ES6+ constructs by default in library code
     # - (various ES6 operators available in all browsers listed below)
     # - https://caniuse.com/mdn-javascript_operators_nullish_coalescing:
@@ -1137,14 +1137,11 @@ def phase_linker_setup(options, state, newargs):
     # Taking the highest requirements gives is our minimum:
     #                             Max Version: FF:79 CHROME:85 SAFARI:14 NODE:16
     # TODO: replace this with feature matrix in the future.
-    settings.TRANSPILE_TO_ES5 = (settings.MIN_FIREFOX_VERSION < 79 or
-                                 settings.MIN_CHROME_VERSION < 85 or
-                                 settings.MIN_SAFARI_VERSION < 140000 or
-                                 settings.MIN_NODE_VERSION < 160000 or
-                                 settings.MIN_IE_VERSION != 0x7FFFFFFF)
-
-    if options.use_closure_compiler is None and settings.TRANSPILE_TO_ES5:
-      diagnostics.warning('transpile', 'enabling transpilation via closure due to browser version settings.  This warning can be suppressed by passing `--closure=1` or `--closure=0` to opt into this explicitly.')
+    settings.TRANSPILE = (settings.MIN_FIREFOX_VERSION < 79 or
+                          settings.MIN_CHROME_VERSION < 85 or
+                          settings.MIN_SAFARI_VERSION < 140000 or
+                          settings.MIN_NODE_VERSION < 160000 or
+                          settings.MIN_IE_VERSION != 0x7FFFFFFF)
 
   # https://caniuse.com/class: FF:45 CHROME:49 SAFARI:9
   supports_es6_classes = (settings.MIN_FIREFOX_VERSION >= 45 and
@@ -2200,14 +2197,15 @@ def phase_binaryen(target, options, wasm_target, memfile):
     with ToolchainProfiler.profile_block('asyncify_lazy_load_code'):
       building.asyncify_lazy_load_code(wasm_target, debug=intermediate_debug_info)
 
-  if final_js and (options.use_closure_compiler or settings.TRANSPILE_TO_ES5):
+  if final_js:
     if options.use_closure_compiler:
       with ToolchainProfiler.profile_block('closure_compile'):
         final_js = building.closure_compiler(final_js, extra_closure_args=options.closure_args)
-    else:
-      with ToolchainProfiler.profile_block('closure_transpile'):
-        final_js = building.closure_transpile(final_js)
-    save_intermediate_with_wasm('closure', wasm_target)
+      save_intermediate_with_wasm('closure', wasm_target)
+    if settings.TRANSPILE:
+      with ToolchainProfiler.profile_block('transpile'):
+        final_js = building.transpile(final_js)
+      save_intermediate_with_wasm('traspile', wasm_target)
 
   symbols_file = None
   if options.emit_symbol_map: