Add ECMAScript test suite (test262)

.github/workflows/test262.yml

@@ -0,0 +1,69 @@
+name: EcmaScript official test suite (test262)
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - v*
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  run_test262:
+    name: Run the test262 test suite
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Install the Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+          profile: minimal
+      - name: Cache cargo registry
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+      - name: Cache cargo index
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+      - name: Cache cargo build
+        uses: actions/cache@v1
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+
+      # Run the test suite and upload the results
+      - name: Checkout GitHub pages
+        if: github.event_name == 'push'
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+          path: gh-pages
+      - run: mkdir -p gh-pages/test262
+
+      - name: Run the test262 test suite
+        run: cargo run --release --bin boa_tester -- -o gh-pages/test262
+
+      - name: Commit files
+        if: github.event_name == 'push'
+        run: |
+          cd gh-pages
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git pull
+          git commit -m "Add new test262 results" -a
+          cd ..
+      - name: Upload results
+        if: github.event_name == 'push'
+        uses: ad-m/[email protected]
+        with:
+          directory: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/webassembly.yml

@@ -16,10 +16,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Check code formatting
-        uses: creyD/[email protected]
-        with:
-          dry: true
-          prettier_options: --check .
+        run: npx prettier --check .
 
   build:
     name: Build webassembly demo
@@ -34,7 +31,7 @@ jobs:
           override: true
           profile: minimal
       - name: Install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh 
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: Cache yarn build
         uses: actions/cache@v2
         with:

.gitignore

@@ -23,4 +23,7 @@ tests/js/test.js
 *.string_data
 *.string_index
 *.events
-chrome_profiler.json 
+chrome_profiler.json
+
+# Logs
+*.log

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test262"]
+	path = test262
+	url = https://github.com/tc39/test262.git

.prettierignore

@@ -6,3 +6,5 @@ boa/benches/bench_scripts/mini_js.js
 boa/benches/bench_scripts/clean_js.js
 boa_wasm/pkg
 dist
+test262
+tests/js/test.js

CONTRIBUTING.md

@@ -65,6 +65,25 @@ There are some pre-defined tasks in [tasks.json](.vscode/tasks.json)
 If you don't want to install everything on your machine, you can use the Dockerfile.
 Start VSCode in container mode (you may need the docker container plugin) and use the Dockerfile.
 
+## Testing
+
+Boa provides its own test suite, and can also run the official ECMAScript test suite. To run the Boa test
+suite, you can just run the normal `cargo test`, and to run the full ECMAScript test suite, you can run it
+with this command:
+
+```
+cargo run --release --bin boa_tester -- -v 2> error.log
+```
+
+Note that this requires the `test262` submodule to be checked out, so you will need to run the following first:
+
+```
+git submodule init && git submodule update
+```
+
+This will run the test suite in verbose mode (you can remove the `-- -v` part to run it in non-verbose mode),
+and output nice colorings in the terminal. It will also output any panic information into the `error.log` file.
+
 ## Communication
 
 We have a Discord server, feel free to ask questions here:

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "boa",
     "boa_cli",
     "boa_wasm",
+    "tester",
 ]
 
 # The release profile, used for `cargo build --release`.

boa/Cargo.toml

@@ -23,7 +23,7 @@ rustc-hash = "1.1.0"
 num-bigint = { version = "0.3.0", features = ["serde"] }
 num-integer = "0.1.43"
 bitflags = "1.2.1"
-indexmap = "1.5.1"
+indexmap = "1.5.2"
 ryu-js = "0.2.0"
 chrono = "0.4.15"
 

boa/src/builtins/array/mod.rs

@@ -655,7 +655,10 @@ impl Array {
         }
 
         let search_element = args[0].clone();
-        let len = this.get_field("length").as_number().unwrap() as i32;
+        let len = this
+            .get_field("length")
+            .as_number()
+            .expect("length was not a number") as i32;
 
         let mut idx = match args.get(1) {
             Some(from_idx_ptr) => {

boa/src/builtins/math/tests.rs

@@ -721,7 +721,7 @@ fn tan() {
     assert!(float_cmp::approx_eq!(
         f64,
         a.to_number(&mut engine).unwrap(),
-        f64::from(1.964_759_657_248_652_5)
+        1.964_759_657_248_652_5
     ));
 }
 

boa/src/exec/tests.rs

@@ -1303,7 +1303,7 @@ fn assignment_to_non_assignable() {
     for case in test_cases.iter() {
         let string = forward(&mut engine, case);
 
-        assert!(string.starts_with("Syntax Error: "));
+        assert!(string.starts_with("Uncaught \"SyntaxError\": "));
         assert!(string.contains("1:3"));
     }
 }
@@ -1318,9 +1318,9 @@ fn multicharacter_assignment_to_non_assignable() {
     let test_cases = ["3 **= 5", "3 <<= 5", "3 >>= 5"];
 
     for case in test_cases.iter() {
-        let string = forward(&mut engine, case);
+        let string = dbg!(forward(&mut engine, case));
 
-        assert!(string.starts_with("Syntax Error: "));
+        assert!(string.starts_with("Uncaught \"SyntaxError\": "));
         assert!(string.contains("1:3"));
     }
 }
@@ -1335,9 +1335,9 @@ fn multicharacter_bitwise_assignment_to_non_assignable() {
     let test_cases = ["3 >>>= 5", "3 &&= 5", "3 ||= 5", "3 ??= 5"];
 
     for case in test_cases.iter() {
-        let string = forward(&mut engine, case);
+        let string = dbg!(forward(&mut engine, case));
 
-        assert!(string.starts_with("Syntax Error: "));
+        assert!(string.starts_with("Uncaught \"SyntaxError\": "));
         assert!(string.contains("1:3"));
     }
 }
@@ -1347,10 +1347,10 @@ fn assign_to_array_decl() {
     let realm = Realm::create();
     let mut engine = Interpreter::new(realm);
 
-    assert!(forward(&mut engine, "[1] = [2]").starts_with("Syntax Error: "));
-    assert!(forward(&mut engine, "[3, 5] = [7, 8]").starts_with("Syntax Error: "));
-    assert!(forward(&mut engine, "[6, 8] = [2]").starts_with("Syntax Error: "));
-    assert!(forward(&mut engine, "[6] = [2, 9]").starts_with("Syntax Error: "));
+    assert!(forward(&mut engine, "[1] = [2]").starts_with("Uncaught \"SyntaxError\": "));
+    assert!(forward(&mut engine, "[3, 5] = [7, 8]").starts_with("Uncaught \"SyntaxError\": "));
+    assert!(forward(&mut engine, "[6, 8] = [2]").starts_with("Uncaught \"SyntaxError\": "));
+    assert!(forward(&mut engine, "[6] = [2, 9]").starts_with("Uncaught \"SyntaxError\": "));
 }
 
 #[test]
@@ -1359,7 +1359,7 @@ fn assign_to_object_decl() {
     let mut engine = Interpreter::new(realm);
 
     const ERR_MSG: &str =
-        "expected token \';\', got \':\' in expression statement at line 1, col 3";
+        "Uncaught \"SyntaxError\": \"expected token \';\', got \':\' in expression statement at line 1, col 3\"";
 
     assert_eq!(forward(&mut engine, "{a: 3} = {a: 5};"), ERR_MSG);
 }