Correct the version of WebAssembly#136 on master (WebAssembly#141)

* Add the WASI repo as a submodule.

Also, add the witx filenames to the generated output, and just have
`cargo run` auto-generate the api.h header, rather than using clap.

* Switch witx to a path dependency.

* Add a test.

* Add a test that the generated file is in sync with the generator.

* Enable CI testing with Github Actions.

* Fix the name of the wasi-headers directory.

* Enable submodules.

* Add a diff mechanism to help explain failures.

* Sort the inputs for display.

* More debugging.

* More debugging.

* Add a .gitattributes file forcing text files to be eol=lf.

Most editors these days can deal with eof=lf files, even on Windows, and
this avoids trouble with headers and other generated files differing in
line endings.

.gitattributes

@@ -0,0 +1,4 @@
+# Se publish headers and other files from the repo, and we don't want
+# them differing depending on the line-ending style of the host they
+# were checked out on.
+* text eol=lf

.github/workflows/main.yml

@@ -0,0 +1,41 @@
+name: CI
+on: [push, pull_request]
+
+jobs:
+  test:
+    name: Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+    - uses: actions/checkout@master
+      with:
+        submodules: true
+    - name: Install Rust (rustup)
+      shell: bash
+      run: rustup update stable --no-self-update && rustup default stable
+      if: matrix.os != 'macos-latest'
+    - name: Install Rust (macos)
+      run: |
+        curl https://sh.rustup.rs | sh -s -- -y
+        echo "##[add-path]$HOME/.cargo/bin"
+      if: matrix.os == 'macos-latest'
+    - run: cargo fetch
+      working-directory: tools/wasi-headers
+    - run: cargo build
+      working-directory: tools/wasi-headers
+    - run: cargo test
+      working-directory: tools/wasi-headers
+
+  rustfmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+      with:
+        submodules: true
+    - name: Install Rust
+      run: rustup update stable && rustup default stable && rustup component add rustfmt
+    - run: cargo fmt -- --check
+      working-directory: tools/wasi-headers

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tools/wasi-headers/WASI"]
+	path = tools/wasi-headers/WASI
+	url = https://github.com/WebAssembly/WASI

libc-bottom-half/headers/public/wasi/api.h

@@ -1,14 +1,16 @@
 /**
- * THIS FILE IS AUTO-GENERATED!
+ * THIS FILE IS AUTO-GENERATED from the following files:
+ *   typenames.witx, wasi_snapshot_preview1.witx
  *
  * @file
- * This file describes the WASI interface, consisting of functions, types,
+ * This file describes the [WASI] interface, consisting of functions, types,
  * and defined values (macros).
  *
  * The interface described here is greatly inspired by [CloudABI]'s clean,
  * thoughtfully-designed, cabability-oriented, POSIX-style API.
  *
  * [CloudABI]: https://github.com/NuxiNL/cloudlibc
+ * [WASI]: https://github.com/WebAssembly/WASI/
  */
 
 #ifndef __wasi_api_h

tools/wasi-headers/.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

tools/wasi-headers/Cargo.toml

@@ -7,6 +7,9 @@ edition = "2018"
 publish = false
 
 [dependencies]
-clap = "2"
 heck = "0.3.1"
-witx = "0.5.0"
+witx = { path = "WASI/tools/witx" }
+anyhow = "1.0.22"
+
+[dev-dependencies]
+diff = "0.1.11"

tools/wasi-headers/src/c_header.rs

@@ -1,17 +1,23 @@
 use heck::ShoutySnakeCase;
 use witx::*;
 
-const PROLOGUE: &str = r#"/**
- * THIS FILE IS AUTO-GENERATED!
+pub(crate) fn to_c_header(doc: &Document, inputs_str: &str) -> String {
+    let mut ret = String::new();
+
+    ret.push_str(&format!(
+        r#"/**
+ * THIS FILE IS AUTO-GENERATED from the following files:
+ *   {}
  *
  * @file
- * This file describes the WASI interface, consisting of functions, types,
+ * This file describes the [WASI] interface, consisting of functions, types,
  * and defined values (macros).
  *
  * The interface described here is greatly inspired by [CloudABI]'s clean,
  * thoughtfully-designed, cabability-oriented, POSIX-style API.
  *
  * [CloudABI]: https://github.com/NuxiNL/cloudlibc
+ * [WASI]: https://github.com/WebAssembly/WASI/
  */
 
 #ifndef __wasi_api_h
@@ -34,23 +40,14 @@ _Static_assert(_Alignof(int64_t) == 8, "non-wasi data layout");
 _Static_assert(_Alignof(uint64_t) == 8, "non-wasi data layout");
 
 #ifdef __cplusplus
-extern "C" {
+extern "C" {{
 #endif
 
 // TODO: Encoding this in witx.
 #define __WASI_DIRCOOKIE_START (UINT64_C(0))
-"#;
-
-const EPILOGUE: &str = r#"#ifdef __cplusplus
-}
-#endif
-
-#endif"#;
-
-pub fn to_c_header(doc: &Document) -> String {
-    let mut ret = String::new();
-
-    ret.push_str(PROLOGUE);
+"#,
+        inputs_str,
+    ));
 
     for d in doc.datatypes() {
         print_datatype(&mut ret, &*d);
@@ -60,7 +57,14 @@ pub fn to_c_header(doc: &Document) -> String {
         print_module(&mut ret, doc, &m);
     }
 
-    ret.push_str(EPILOGUE);
+    ret.push_str(
+        r#"#ifdef __cplusplus
+}
+#endif
+
+#endif
+"#,
+    );
 
     ret
 }

tools/wasi-headers/src/lib.rs

@@ -0,0 +1,26 @@
+mod c_header;
+
+use anyhow::{anyhow, Result};
+use c_header::to_c_header;
+use std::fs::read_dir;
+use std::io;
+use witx::load;
+
+pub fn generate() -> Result<String> {
+    let mut inputs = read_dir("WASI/phases/snapshot/witx")?
+        .map(|res| res.map(|e| e.path()))
+        .collect::<Result<Vec<_>, io::Error>>()?;
+
+    inputs.sort();
+
+    // TODO: drop the anyhow! part once witx switches to anyhow.
+    let doc = load(&inputs).map_err(|e| anyhow!(e.to_string()))?;
+
+    let inputs_str = &inputs
+        .iter()
+        .map(|p| p.file_name().unwrap().to_str().unwrap().to_string())
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    Ok(to_c_header(&doc, &inputs_str))
+}

tools/wasi-headers/src/main.rs

@@ -1,59 +1,11 @@
-mod c_header;
-
-use crate::c_header::to_c_header;
-use clap::{App, Arg};
+use anyhow::Result;
 use std::fs::File;
 use std::io::Write;
-use std::path::PathBuf;
-use std::process;
-use witx::load;
-
-pub fn main() {
-    let app = App::new("wasi-headers")
-        .version(env!("CARGO_PKG_VERSION"))
-        .about("Generate C headers for WASI interfaces")
-        .arg(
-            Arg::with_name("input")
-                .required(true)
-                .multiple(true)
-                .help("path to root of witx document"),
-        )
-        .arg(
-            Arg::with_name("verbose")
-                .short("v")
-                .long("verbose")
-                .takes_value(false)
-                .required(false),
-        )
-        .get_matches();
-
-    let inputs = app
-        .values_of("input")
-        .expect("at least one input required")
-        .map(PathBuf::from)
-        .collect::<Vec<PathBuf>>();
-
-    match load(&inputs) {
-        Ok(doc) => {
-            if app.is_present("verbose") {
-                println!("{:?}", doc)
-            }
+use wasi_headers::generate;
 
-            let c_header = to_c_header(&doc);
-            if let Some(output) = app.value_of("output") {
-                let mut file = File::create(output).expect("create output file");
-                file.write_all(c_header.as_bytes())
-                    .expect("write output file");
-            } else {
-                println!("{}", c_header)
-            }
-        }
-        Err(e) => {
-            println!("{}", e.report());
-            if app.is_present("verbose") {
-                println!("{:?}", e);
-            }
-            process::exit(1)
-        }
-    }
+pub fn main() -> Result<()> {
+    let c_header = generate()?;
+    let mut file = File::create("../../libc-bottom-half/headers/public/wasi/api.h")?;
+    file.write_all(c_header.as_bytes())?;
+    Ok(())
 }

tools/wasi-headers/tests/verify.rs

@@ -0,0 +1,69 @@
+#[test]
+fn assert_same_as_src() {
+    let actual = include_str!("../../../libc-bottom-half/headers/public/wasi/api.h");
+    let expected = wasi_headers::generate().expect("header generation should succeed");
+    if actual == expected {
+        return;
+    }
+
+    eprintln!("The following diff was found between the generated <wasi/api.h> and the");
+    eprintln!("source <wasi/api.h> in the tree:");
+    eprintln!();
+
+    let mut expected_line = 1;
+    let mut actual_line = 1;
+    let mut separated = false;
+    let mut any_lines = false;
+    for diff in diff::lines(&expected, &actual) {
+        match diff {
+            diff::Result::Left(l) => {
+                eprintln!("line {}: -{}", expected_line, l);
+                expected_line += 1;
+                separated = false;
+                any_lines = true;
+            }
+            diff::Result::Both(_, _) => {
+                expected_line += 1;
+                actual_line += 1;
+                if !separated {
+                    eprintln!("...");
+                    separated = true;
+                }
+            }
+            diff::Result::Right(r) => {
+                eprintln!("line {}: +{}", actual_line, r);
+                actual_line += 1;
+                separated = false;
+                any_lines = true;
+            }
+        }
+    }
+
+    if !any_lines {
+        eprintln!();
+        eprintln!(
+            "Somehow there was a diff with no lines differing. Lengths: {} and {}.",
+            expected.len(),
+            actual.len()
+        );
+        for (index, (a, b)) in actual.chars().zip(expected.chars()).enumerate() {
+            if a != b {
+                eprintln!("char difference at index {}: '{}' != '{}'", index, a, b);
+            }
+        }
+        for (index, (a, b)) in actual.bytes().zip(expected.bytes()).enumerate() {
+            if a != b {
+                eprintln!("byte difference at index {}: b'{}' != b'{}'", index, a, b);
+            }
+        }
+        eprintln!();
+        eprintln!("actual: {}", actual);
+        eprintln!();
+        eprintln!("expected: {}", expected);
+    }
+
+    eprintln!();
+    eprintln!("To regenerate the files, run `cd tools/wasi-headers && cargo run`.");
+    eprintln!();
+    panic!();
+}