diff --git a/.github/workflows/reproducibility.yaml b/.github/workflows/reproducibility.yaml index e1a1835bb6c..b9cd05834d7 100644 --- a/.github/workflows/reproducibility.yaml +++ b/.github/workflows/reproducibility.yaml @@ -45,14 +45,17 @@ jobs: # Build artifacts that are supposed to be reproducible. - name: Build Rust server - run: ./scripts/docker_run ./scripts/build_server -s base + run: | + ./scripts/docker_run ./scripts/runner build-server --server-variant=base - name: Build examples - run: ./scripts/docker_run ./scripts/build_examples + run: | + ./scripts/docker_run ./scripts/runner run-examples --run-clients=false --run-server=false # Generate an index of the hashes of the reproducible artifacts. - name: Generate Reproducibility Index - run: ./scripts/docker_run ./scripts/build_reproducibility_index + run: | + ./scripts/docker_run ./scripts/build_reproducibility_index # Remove all files from the "out" folder. - name: Clear "out" folder diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 38b0ecc9fdf..5fa5be1377a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -44,7 +44,15 @@ steps: waitFor: ['git_init'] timeout: 60m entrypoint: 'bash' - args: ['./scripts/build_example', '-e', 'hello_world', '-i', 'base'] + args: + [ + './scripts/runner', + 'run-example', + '--example-name=hello_world', + '--run-server=false', + '--run-clients=false', + '--build-docker', + ] - name: 'gcr.io/oak-ci/oak:latest' id: generate_root_ca_certs diff --git a/docs/INSTALL-OSX.md b/docs/INSTALL-OSX.md index 7b02d250402..61e43b9d85b 100644 --- a/docs/INSTALL-OSX.md +++ b/docs/INSTALL-OSX.md @@ -62,20 +62,20 @@ retrieved from a Google Cloud Docker repository. The Oak Runtime and its dependencies are built with the following script: ```bash -./scripts/build_server +./scripts/runner build-server ``` Build a particular example, say `hello_world`, with: ```bash -./scripts/build_example -e hello_world +./scripts/runner run-exammples --example-name=hello_world ``` Note that the Runtime server requires a particular Oak Application to run, and so relies on the previous section. ```bash -./scripts/run_server -e hello_world +./scripts/runner run-examples --run-clients=false --example-name=hello_world ``` In a separate terminal, run an example client that connects to the Oak Runtime @@ -84,7 +84,7 @@ needed, so the client will connect to the already-running server of the previous step): ```bash -./scripts/run_example -s none -e hello_world +./scripts/runner run-examples --run-server=false --example-name=hello_world ``` ## Codebase Tools diff --git a/docs/development.md b/docs/development.md index d80ccf5e2c5..dd86be85cda 100644 --- a/docs/development.md +++ b/docs/development.md @@ -16,7 +16,7 @@ To build and run one of the Oak example applications under Docker, run: ```bash ./scripts/docker_pull # retrieve cached Docker image for faster builds -./scripts/docker_run ./scripts/run_example -e hello_world +./scripts/docker_run ./scripts/runner run-examples --example-name=hello_world ``` This should build the Runtime, an Oak Application and a client for the @@ -118,13 +118,13 @@ Running one of the example Oak applications will confirm that all core prerequisites have been installed. Run one inside Docker with: ```bash -./scripts/docker_run ./scripts/run_example -e hello_world +./scripts/docker_run ./scripts/runner run-examples --example-name=hello_world ``` or, if all prerequisites are available on the local system, outside of Docker: ```bash -./scripts/run_example -e hello_world +./scripts/runner run-examples --example-name=hello_world ``` That script: @@ -151,7 +151,7 @@ to a WebAssembly module and then serializes it into a binary application configuration file to be loaded to the Oak Server: ```bash -./scripts/build_example -e hello_world +./scripts/runner run-examples --run-server=false --run-clients=false --example-name=hello_world ``` This binary application configuration file includes the compiled Wasm code for @@ -172,7 +172,7 @@ The following command builds the Oak Runtime server. An initial build will take some time, but subsequent builds should be cached and so run much faster. ```bash -./scripts/build_server +./scripts/runner build-server ``` ### Run Runtime Server @@ -182,7 +182,7 @@ Oak Application (which must already have been compiled into WebAssembly and built into a serialized configuration, as [described above](#build-application). ```bash -./scripts/run_server -e hello_world +./scripts/runner run-examples --run-clients=false --example-name=hello_world ``` In the end, you should end up with an Oak server running, end with log output @@ -202,7 +202,7 @@ client of an example Oak Application (as [described above](#build-application)), and runs the client code locally. ```bash -./scripts/run_example -s none -e hello_world +./scripts/runner run-examples --run-server=false --logs --example-name=hello_world ``` The `-s none` option indicates that the script expects to find an diff --git a/docs/programming-oak.md b/docs/programming-oak.md index 3db956416f2..614bf809d3d 100644 --- a/docs/programming-oak.md +++ b/docs/programming-oak.md @@ -262,17 +262,17 @@ Here: configuration. All these steps are implemented as a part of the -`./scripts/build_example -e hello_world` script. +`./scripts/runner run-examples --example-name=hello_world` script. ### Starting the Oak Application -The Oak Application is then loaded using the Oak Runner: +The Oak Application is then started using the Oak Loader: ```bash -./scripts/run_server -f "${PWD}/config.bin" +./oak/server/target/x86_64-unknown-linux-musl/release/oak_loader --application="${PWD}/config.bin" ``` -The Oak Runner will launch an [Oak Runtime](concepts.md#oak-runtime), and this +The Oak Loader will launch an [Oak Runtime](concepts.md#oak-runtime), and this Runtime will check the provided Wasm module(s) and application configuration. Assuming everything is correct (e.g. the Nodes all have a main entrypoint and only expect to link to the Oak [host functions](abi.md#host-functions)), the Oak diff --git a/examples/abitest/config/build.sh b/examples/abitest/config/build.sh index 9f557655012..97297938fad 100644 --- a/examples/abitest/config/build.sh +++ b/examples/abitest/config/build.sh @@ -7,5 +7,5 @@ mkdir --parents "${OUT_DIR}" bazel run //oak/common:app_config_serializer -- \ --textproto="${DIR}/config.textproto"\ - --modules="frontend_module:${MODULES_DIR}/abitest_0_frontend.wasm","backend_module:${MODULES_DIR}/abitest_1_backend.wasm" \ + --modules="frontend_module:${MODULES_DIR}/abitest_0_frontend.wasm,backend_module:${MODULES_DIR}/abitest_1_backend.wasm" \ --output_file="${OUT_DIR}/config.bin" diff --git a/examples/aggregator/README.md b/examples/aggregator/README.md index c6d9f9808de..a1d36fac254 100644 --- a/examples/aggregator/README.md +++ b/examples/aggregator/README.md @@ -37,13 +37,7 @@ a Sparse Vector - a dictionary with integer keys. Build and run the Aggregator with the following command: ```bash -./scripts/build_example -e aggregator -./scripts/build_server -s base -cargo run --release --target=x86_64-unknown-linux-musl --manifest-path=oak/server/rust/oak_loader/Cargo.toml -- \ - --application=./examples/aggregator/bin/config.bin \ - --grpc-tls-private-key=./examples/certs/local/local.key \ - --grpc-tls-certificate=./examples/certs/local/local.pem \ - --root-tls-certificate=./examples/certs/local/ca.pem +./scripts/runner run-examples --run-clients=false --example-name=aggregator ``` Aggregator code is in `common` and `module` directories (where `common` defines @@ -56,12 +50,7 @@ Simple client that connects to the Aggregator and sends a data sample via gRPC. Build and run the Client with the following command: ```bash -./scripts/build_example -e aggregator -./bazel-client-bin/examples/aggregator/client/client \ - --address=localhost:8080 \ - --ca_cert=./examples/certs/local/ca.pem \ - --bucket=test \ - --data=1:10,2:20,3:30 +./scripts/runner run-examples --run-server=false --example-name=aggregator ``` A common use case is to keep running the client until the aggregation threshold diff --git a/examples/aggregator/scripts/docker_push b/examples/aggregator/scripts/docker_push index 2677c05f450..94b8b9ff41d 100755 --- a/examples/aggregator/scripts/docker_push +++ b/examples/aggregator/scripts/docker_push @@ -1,21 +1,20 @@ #!/usr/bin/env bash readonly GLOBAL_SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")/../../../scripts/" -readonly SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")" # shellcheck source=scripts/common source "${GLOBAL_SCRIPTS_DIR}/common" readonly AGGREGATOR_DOCKER_IMAGE_NAME='gcr.io/oak-ci/oak-aggregator' readonly BACKEND_DOCKER_IMAGE_NAME='gcr.io/oak-ci/oak-aggregator-backend' -declare -a image backend_image +declare image backend_image image="$(find ./examples/aggregator/bin -name aggregator.tar)" backend_image="$(find ./examples/aggregator/bin -name aggregator_backend.tar)" if [[ -z ${image} || -z ${backend_image} ]]; then echo "Docker images have not been built, run './scripts/build_example -e aggregator -i'" exit 1 fi -if [[ ${#image[@]} > 1 || ${#backend_image[@]} > 1 ]]; then +if [[ ${#image[@]} -gt 1 || ${#backend_image[@]} -gt 1 ]]; then echo "Too many Docker images: ${image}, ${backend_image}" exit 0 fi diff --git a/examples/aggregator/scripts/run_backend b/examples/aggregator/scripts/run_backend index e5f86737e89..53f7078db34 100755 --- a/examples/aggregator/scripts/run_backend +++ b/examples/aggregator/scripts/run_backend @@ -1,7 +1,6 @@ #!/usr/bin/env bash readonly GLOBAL_SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")/../../../scripts/" -readonly SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")" # shellcheck source=scripts/common source "${GLOBAL_SCRIPTS_DIR}/common" diff --git a/examples/aggregator/scripts/run_client b/examples/aggregator/scripts/run_client index 60b490de271..74546d6244f 100755 --- a/examples/aggregator/scripts/run_client +++ b/examples/aggregator/scripts/run_client @@ -1,7 +1,6 @@ #!/usr/bin/env bash readonly GLOBAL_SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")/../../../scripts/" -readonly SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")" # shellcheck source=scripts/common source "${GLOBAL_SCRIPTS_DIR}/common" diff --git a/examples/aggregator/scripts/run_server b/examples/aggregator/scripts/run_server index 0f90a32ab01..4970602c9a7 100755 --- a/examples/aggregator/scripts/run_server +++ b/examples/aggregator/scripts/run_server @@ -1,7 +1,6 @@ #!/usr/bin/env bash readonly GLOBAL_SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")/../../../scripts/" -readonly SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")" # shellcheck source=scripts/common source "${GLOBAL_SCRIPTS_DIR}/common" diff --git a/examples/aggregator/scripts/undeploy b/examples/aggregator/scripts/undeploy index bb1c4c0d1ad..2553c48291b 100755 --- a/examples/aggregator/scripts/undeploy +++ b/examples/aggregator/scripts/undeploy @@ -1,7 +1,6 @@ #!/usr/bin/env bash readonly GLOBAL_SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")/../../../scripts/" -readonly SCRIPTS_DIR="$(dirname "$(readlink -f "$0")")" # shellcheck source=scripts/common source "${GLOBAL_SCRIPTS_DIR}/common" @@ -11,7 +10,7 @@ while getopts "mh" opt; do case "${opt}" in h) echo -e "Usage: ${0} [-m] -Undeploy aggregator objects. +Undeploy aggregator objects. -m In addition, undeploy metrics related objects. -h Print Help (this message) and exit" exit 0;; @@ -27,9 +26,9 @@ gcloud container clusters get-credentials aggregator --zone=europe-west2-a kubectl delete secret tls-private-key || true kubectl delete service aggregator || true -kubectl delete deployment aggregator || true +kubectl delete deployment aggregator || true -if [[ $metrics == 1]] +if [[ $metrics == 1 ]] then kubectl delete service prometheus-k8s -n agg-metrics || true kubectl delete deployment metrics-sidecar -n agg-metrics || true diff --git a/examples/chat/README.md b/examples/chat/README.md index 7f1ae3331d4..ca6f0efb2c1 100644 --- a/examples/chat/README.md +++ b/examples/chat/README.md @@ -15,23 +15,17 @@ as bearer tokens: ## Command Line Operation -Initially, the chat example needs to be built: +Initially, the chat example needs to be built, and the server started: ```bash -./scripts/build_example -e chat -``` - -Next, let's start the server: - -```bash -./scripts/run_server -e chat +./scripts/runner --logs run-examples --run-clients=false --example-name=chat ``` After this, one can run the first client, which connects to the Oak Application and opens a first chat room inside it: ```bash -./scripts/run_example -s none -e chat +./scripts/runner --logs run-examples --run-server=false --example-name=chat --client-additional-args=--test=false ``` This will emit a trace line that holds the information needed to: @@ -46,22 +40,7 @@ This will emit a trace line that holds the information needed to: Another party can then join the same chat room by using these arguments: ```bash -./scripts/run_example -s none -e chat -- --address=localhost:8080 --room_id=NKsceNlg69UbcvryfzmFGnMv9qnZ0DYh6u6gJxujnPPxvHsxMehoD368sumKawVaq9WaSkzrcStoNYLvVNdzhA== -``` - -Alternatively, another party can create a new chat room running on the same Oak -Application by just copying the `--address` argument, but specifying a new room -name: - -```bash -./scripts/run_example -s none -e chat -- --address=localhost:8080 -``` - -This will again emit a trace line with the information needed to join this new -room (on the same server): - -```log -2019-10-24 11:04:40 INFO chat.cc : 242 : Join this room with --address=localhost:8080 --room_id=msSGas1Ie2rtGIvG0bLa2Jh3ODjO35nix46R3j2iYjAcB8zDcJpn/P2DD7c0yB1NMmfoipBSAePJzlXjknm8gg== +./scripts/runner --logs run-examples --run-server=false --example-name=chat --client-additional-args=--test=false --client-additional-args=--room_id=NKsceNlg69UbcvryfzmFGnMv9qnZ0DYh6u6gJxujnPPxvHsxMehoD368sumKawVaq9WaSkzrcStoNYLvVNdzhA== ``` ## CI Invocation diff --git a/examples/hello_world/client/nodejs/README.MD b/examples/hello_world/client/nodejs/README.MD index 74d987cb407..4eca4d845c7 100644 --- a/examples/hello_world/client/nodejs/README.MD +++ b/examples/hello_world/client/nodejs/README.MD @@ -1,7 +1,10 @@ # Node.js client example Run this example with the following command: -`./scripts/docker_run ./scripts/run_example -e hello_world -c npm` + +```bash +./scripts/docker_run --logs ./scripts/runner run-examples --example-name=hello_world +``` If everything works smoothly, you should then find `HELLO Node.js!` logged to -the console, alongside logs from the Oak application. +the console, alongside logs from the Oak application and from other clients. diff --git a/examples/hello_world/config/build.sh b/examples/hello_world/config/build.sh index be69e090f60..15f8623e2df 100644 --- a/examples/hello_world/config/build.sh +++ b/examples/hello_world/config/build.sh @@ -7,5 +7,5 @@ mkdir --parents "${OUT_DIR}" bazel run //oak/common:app_config_serializer -- \ --textproto="${DIR}/config.textproto"\ - --modules="app:${MODULES_DIR}/hello_world.wasm","translator:${MODULES_DIR}/translator.wasm" \ + --modules="app:${MODULES_DIR}/hello_world.wasm,translator:${MODULES_DIR}/translator.wasm" \ --output_file="${OUT_DIR}/config.bin" diff --git a/examples/hello_world/example.toml b/examples/hello_world/example.toml index 6610d8cb6db..9f62f6fd963 100644 --- a/examples/hello_world/example.toml +++ b/examples/hello_world/example.toml @@ -6,8 +6,10 @@ out = "examples/hello_world/bin/config.bin" [modules] module = { Cargo = { cargo_manifest = "examples/hello_world/module/rust/Cargo.toml" } } +# Disabled by default, since it requires emsdk to be installed on the host. +# module_cpp = { Bazel = { bazel_target = "//examples/hello_world/module/cpp:all", config = "emscripten" } } translator = { Cargo = { cargo_manifest = "examples/translator/module/rust/Cargo.toml" } } [clients] cpp = { Bazel = { bazel_target = "//examples/hello_world/client:client" } } -# nodejs = { Npm = { package_directory = "examples/hello_world/client/nodejs" } } +nodejs = { Npm = { package_directory = "examples/hello_world/client/nodejs" } } diff --git a/examples/private_set_intersection/example.toml b/examples/private_set_intersection/example.toml new file mode 100644 index 00000000000..154eeebd9b1 --- /dev/null +++ b/examples/private_set_intersection/example.toml @@ -0,0 +1,11 @@ +name = "private_set_intersection" + +[application] +build = { Shell = { script = "examples/private_set_intersection/config/build.sh" } } +out = "examples/private_set_intersection/bin/config.bin" + +[modules] +module = { Cargo = { cargo_manifest = "examples/private_set_intersection/module/rust/Cargo.toml" } } + +[clients] +cpp = { Bazel = { bazel_target = "//examples/private_set_intersection/client:client" } } diff --git a/examples/running_average/example.toml b/examples/running_average/example.toml new file mode 100644 index 00000000000..fc9c93bc599 --- /dev/null +++ b/examples/running_average/example.toml @@ -0,0 +1,11 @@ +name = "running_average" + +[application] +build = { Shell = { script = "examples/running_average/config/build.sh" } } +out = "examples/running_average/bin/config.bin" + +[modules] +module = { Cargo = { cargo_manifest = "examples/running_average/module/rust/Cargo.toml" } } + +[clients] +cpp = { Bazel = { bazel_target = "//examples/running_average/client:client" } } diff --git a/examples/tensorflow/config/BUILD b/examples/tensorflow/config/BUILD deleted file mode 100644 index 207c516b481..00000000000 --- a/examples/tensorflow/config/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright 2020 The Project Oak Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//oak/common:app_config.bzl", "serialized_config") - -package( - default_visibility = ["//examples/tensorflow:__subpackages__"], - licenses = ["notice"], -) - -exports_files(srcs = glob(["*.textproto"])) - -serialized_config( - name = "config", - modules = { - "app": "//examples/tensorflow/module/cpp:tensorflow_micro.wasm", - }, - textproto = ":config.textproto", -) diff --git a/examples/translator/example.toml b/examples/translator/example.toml new file mode 100644 index 00000000000..11d0ffb74e8 --- /dev/null +++ b/examples/translator/example.toml @@ -0,0 +1,11 @@ +name = "translator" + +[application] +build = { Shell = { script = "examples/translator/config/build.sh" } } +out = "examples/translator/bin/config.bin" + +[modules] +module = { Cargo = { cargo_manifest = "examples/translator/module/rust/Cargo.toml" } } + +[clients] +cpp = { Bazel = { bazel_target = "//examples/translator/client:client" } } diff --git a/examples/trusted_information_retrieval/config/build.sh b/examples/trusted_information_retrieval/config/build.sh index f783f809e68..c9774a792f5 100644 --- a/examples/trusted_information_retrieval/config/build.sh +++ b/examples/trusted_information_retrieval/config/build.sh @@ -7,5 +7,5 @@ mkdir --parents "${OUT_DIR}" bazel run //oak/common:app_config_serializer -- \ --textproto="${DIR}/config.textproto"\ - --modules="app:${MODULES_DIR}/trusted_information_retrieval.wasm","database_proxy:${MODULES_DIR}/database_proxy.wasm"\ + --modules="app:${MODULES_DIR}/trusted_information_retrieval.wasm,database_proxy:${MODULES_DIR}/database_proxy.wasm"\ --output_file="${OUT_DIR}/config.bin" diff --git a/runner/src/check_license.rs b/runner/src/check_license.rs index 13d2802f34d..6b621992df0 100644 --- a/runner/src/check_license.rs +++ b/runner/src/check_license.rs @@ -15,6 +15,7 @@ // use super::*; + /// A [`Runnable`] command that checks for the existence of source files without the necessary /// license header. pub struct CheckLicense { diff --git a/runner/src/internal.rs b/runner/src/internal.rs index ef5403b57c3..8e07dd6eac7 100644 --- a/runner/src/internal.rs +++ b/runner/src/internal.rs @@ -71,6 +71,8 @@ pub struct RunExamples { pub client_additional_args: Vec, #[structopt(long, help = "additional arguments to pass to server")] pub server_additional_args: Vec, + #[structopt(long, help = "build a Docker image for the examples")] + pub build_docker: bool, } #[derive(StructOpt, Clone)] @@ -322,6 +324,11 @@ impl Runnable for Cmd { cmd.env("USER", v); } + // Python variables. + if let Ok(v) = std::env::var("PYTHONPATH") { + cmd.env("PYTHONPATH", v); + } + // Rust compilation variables. if let Ok(v) = std::env::var("RUSTUP_HOME") { cmd.env("RUSTUP_HOME", v); @@ -377,14 +384,22 @@ impl Runnable for Cmd { .spawn() .expect("could not spawn command"); - let stdout = read_background(child.stdout.take().expect("could not take stdout")); - let stderr = read_background(child.stderr.take().expect("could not take stderr")); + let stdout_logs = if opt.logs { + Default::default() + } else { + read_background(child.stdout.take().expect("could not take stdout")) + }; + let stderr_logs = if opt.logs { + Default::default() + } else { + read_background(child.stderr.take().expect("could not take stderr")) + }; Box::new(RunningCmd { command: command_string, process: Some(child), - stdout, - stderr, + stdout: stdout_logs, + stderr: stderr_logs, }) } } diff --git a/runner/src/main.rs b/runner/src/main.rs index 541dc246d25..02cc8538ea7 100644 --- a/runner/src/main.rs +++ b/runner/src/main.rs @@ -231,6 +231,7 @@ fn run_tests() -> Step { steps: vec![ run_cargo_clippy(), run_cargo_test(), + run_cargo_doc(), run_cargo_test_tsan(), run_bazel_build(), run_bazel_test(), @@ -242,6 +243,7 @@ fn format() -> Step { Step::Multiple { name: "format".to_string(), steps: vec![ + run_clang_format(FormatMode::Fix), run_buildifier(FormatMode::Fix), run_prettier(FormatMode::Fix), run_markdownlint(FormatMode::Fix), @@ -257,12 +259,15 @@ fn check_format() -> Step { steps: vec![ run_check_license(), run_check_todo(), + run_clang_format(FormatMode::Check), run_buildifier(FormatMode::Check), run_prettier(FormatMode::Check), run_markdownlint(FormatMode::Check), run_embedmd(FormatMode::Check), run_liche(), run_cargo_fmt(FormatMode::Check), + run_hadolint(), + run_shellcheck(), ], } } @@ -291,6 +296,7 @@ fn run_ci() -> Step { run_clients: None, client_additional_args: Vec::new(), server_additional_args: Vec::new(), + build_docker: false, build_server: BuildServer { server_variant: "base".to_string(), server_rust_toolchain: None, @@ -476,6 +482,11 @@ fn run_example(opt: &RunExamples, example: &Example) -> Step { // faster. build_server(&opt.build_server), ], + if opt.build_docker { + vec![build_docker(&example)] + } else { + vec![] + }, match &example.backend { Some(backend) => vec![Step::Single { name: "build backend".to_string(), @@ -494,6 +505,38 @@ fn run_example(opt: &RunExamples, example: &Example) -> Step { } } +fn build_docker(example: &Example) -> Step { + Step::Multiple { + name: "docker".to_string(), + steps: vec![ + Step::Single { + name: "build server image".to_string(), + command: Cmd::new( + "docker", + &[ + "build", + "--tag=oak_docker", + "--file=./oak/server/Dockerfile", + "./oak/server", + ], + ), + }, + Step::Single { + name: "build example image".to_string(), + command: Cmd::new( + "docker", + &[ + "build", + &format!("--tag={}", example.name), + "--file=./examples/Dockerfile", + &format!("./examples/{}", example.name), + ], + ), + }, + ], + } +} + fn build(target: &Target) -> Box { match target { Target::Cargo { cargo_manifest } => Cmd::new( @@ -648,6 +691,12 @@ fn is_source_code_file(path: &PathBuf) -> bool { || filename.ends_with(".proto") } +/// Return whether the provided path refers to a source file that can be formatted by clang-tidy. +fn is_clang_format_file(path: &PathBuf) -> bool { + let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + filename.ends_with(".cc") || filename.ends_with(".h") || filename.ends_with(".proto") +} + /// Return whether the provided path refers to a Bazel file (`BUILD`, `WORKSPACE`, or `*.bzl`) fn is_bazel_file(path: &PathBuf) -> bool { let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); @@ -660,6 +709,11 @@ fn is_markdown_file(path: &PathBuf) -> bool { filename.ends_with(".md") } +fn is_dockerfile(path: &PathBuf) -> bool { + let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + filename.ends_with("Dockerfile") +} + fn is_toml_file(path: &PathBuf) -> bool { let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); filename.ends_with(".toml") @@ -707,6 +761,19 @@ fn is_cargo_workspace_file(path: &PathBuf) -> bool { contents.contains("[workspace]") } +fn is_shell_script(path: &PathBuf) -> bool { + if path.is_file() { + let mut file = std::fs::File::open(path).expect("could not open file"); + let mut contents = String::new(); + match file.read_to_string(&mut contents) { + Ok(_size) => contents.starts_with("#!"), + Err(_err) => false, + } + } else { + false + } +} + enum FormatMode { Check, Fix, @@ -828,6 +895,64 @@ fn run_liche() -> Step { } } +fn run_hadolint() -> Step { + Step::Multiple { + name: "hadolint".to_string(), + steps: source_files() + .filter(is_dockerfile) + .map(to_string) + .map(|entry| Step::Single { + name: entry.clone(), + command: Cmd::new("hadolint", &[entry]), + }) + .collect(), + } +} + +fn run_shellcheck() -> Step { + Step::Multiple { + name: "shellcheck".to_string(), + steps: source_files() + .filter(is_shell_script) + .map(to_string) + .map(|entry| Step::Single { + name: entry.clone(), + command: Cmd::new("shellcheck", &["--external-sources", &entry]), + }) + .collect(), + } +} + +fn run_clang_format(mode: FormatMode) -> Step { + match mode { + FormatMode::Check => Step::Single { + name: "clang format".to_string(), + command: Cmd::new( + "python", + &[ + "./third_party/run-clang-format/run-clang-format.py", + "-r", + "--exclude", + "*/node_modules", + "oak", + "examples", + ], + ), + }, + FormatMode::Fix => Step::Multiple { + name: "clang format".to_string(), + steps: source_files() + .filter(is_clang_format_file) + .map(to_string) + .map(|entry| Step::Single { + name: entry.clone(), + command: Cmd::new("clang-format", &["-i", "-style=file", &entry]), + }) + .collect(), + }, + } +} + fn run_check_license() -> Step { Step::Multiple { name: "check license".to_string(), @@ -893,6 +1018,13 @@ fn run_cargo_test() -> Step { } } +fn run_cargo_doc() -> Step { + Step::Single { + name: "cargo doc".to_string(), + command: Cmd::new("bash", &["./scripts/check_docs"]), + } +} + fn run_cargo_test_tsan() -> Step { Step::Single { name: "cargo test (tsan)".to_string(),