feat: introduce small object disk cache #52
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CI" | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "forks/*" | |
| pull_request: | |
| branches: | |
| - "main" | |
| - "v*.*.*-rc" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| RUST_TOOLCHAIN_NIGHTLY: nightly-2024-07-19 | |
| CARGO_TERM_COLOR: always | |
| CACHE_KEY_SUFFIX: 20240821 | |
| jobs: | |
| misc-check: | |
| name: misc check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run typos check | |
| uses: crate-ci/typos@master | |
| - name: Install yq | |
| run: | | |
| wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq | |
| env: | |
| YQ_VERSION: v4.16.1 | |
| BINARY: yq_linux_amd64 | |
| BUF_VERSION: 1.0.0-rc6 | |
| - name: Install jq | |
| uses: dcarbone/[email protected] | |
| - name: Check if Grafana dashboards are minimized | |
| run: | | |
| ./scripts/minimize-dashboards.sh --check | |
| - name: Run ShellCheck | |
| uses: ludeeus/action-shellcheck@master | |
| rust-udeps: | |
| name: rust udeps test | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: nightly | |
| - name: Cache Cargo home | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-udeps | |
| - name: Install NASM for aws-lc-rs on Windows | |
| if: runner.os == 'Windows' | |
| uses: ilammy/setup-nasm@v1 | |
| - name: Install ninja-build tool for aws-lc-fips-sys on Windows | |
| if: runner.os == 'Windows' | |
| uses: seanmiddleditch/gha-setup-ninja@v5 | |
| - name: Install cargo-udeps | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| cargo install cargo-udeps --locked | |
| - name: Unused Dependencies Check | |
| env: | |
| RUSTFLAGS: "--cfg tokio_unstable -Awarnings" | |
| run: | | |
| cargo udeps --all-targets | |
| rust-ffmt-check: | |
| name: rust ffmt check | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: nightly | |
| components: rustfmt | |
| - name: Cache Cargo home | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-ffmt-check | |
| - name: Fastidious Format Check | |
| run: | | |
| cargo fmt --all --check -- --config-path rustfmt.nightly.toml | |
| - name: Hint | |
| if: ${{ failure() }} | |
| run: |- | |
| echo "The ffmt (Fastidious Format Check) test is not a necessary." | |
| echo "It uses unstable features to achieve a better format." | |
| echo "If you want to pass the test, please install the nightly toolchain with \`rustup install nightly\`." | |
| echo "Then run \`make ffmt\`." | |
| rust-test: | |
| name: rust test with codecov | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| rust_toolchain: [stable, 1.81.0] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ matrix.rust_toolchain }} | |
| components: rustfmt, clippy, llvm-tools-preview | |
| - name: Cache Cargo home | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-test | |
| - name: Install cargo tools | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| cargo install cargo-sort --locked | |
| - name: Install NASM for aws-lc-rs on Windows | |
| if: runner.os == 'Windows' | |
| uses: ilammy/setup-nasm@v1 | |
| - name: Install ninja-build tool for aws-lc-fips-sys on Windows | |
| if: runner.os == 'Windows' | |
| uses: seanmiddleditch/gha-setup-ninja@v5 | |
| - name: Run rust cargo-sort check | |
| # https://github.com/DevinR528/cargo-sort/issues/56 | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| cargo sort -w -c | |
| - name: Run rust format check | |
| run: | | |
| cargo fmt --all -- --check | |
| - name: Run rust clippy check | |
| run: | | |
| cargo clippy --all-targets --features tokio-console -- -D warnings | |
| cargo clippy --all-targets --features deadlock -- -D warnings | |
| cargo clippy --all-targets --features mtrace -- -D warnings | |
| cargo clippy --all-targets -- -D warnings | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| uses: taiki-e/install-action@nextest | |
| - name: Run rust test with coverage (igored tests) | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CI: true | |
| run: | | |
| cargo llvm-cov --no-report nextest --run-ignored ignored-only --no-capture --workspace --features "strict_assertions,sanity" | |
| - name: Run rust test with coverage | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CI: true | |
| run: | | |
| cargo llvm-cov --no-report nextest --features "strict_assertions,sanity" | |
| - name: Run examples with coverage | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CI: true | |
| run: | | |
| cargo llvm-cov --no-report run --example memory | |
| cargo llvm-cov --no-report run --example hybrid | |
| cargo llvm-cov --no-report run --example hybrid_full | |
| cargo llvm-cov --no-report run --example event_listener | |
| cargo llvm-cov --no-report run --features "mtrace,jaeger" --example tail_based_tracing | |
| cargo llvm-cov --no-report run --features "mtrace,ot" --example tail_based_tracing | |
| - name: Run foyer-bench with coverage | |
| if: runner.os == 'Linux' | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CI: true | |
| run: | | |
| mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov | |
| cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60 | |
| - name: Generate codecov report | |
| run: | | |
| cargo llvm-cov report --lcov --output-path lcov.info | |
| - uses: codecov/codecov-action@v4 | |
| if: runner.os == 'Linux' && matrix.rust_toolchain == 'stable' | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| verbose: true | |
| deadlock: | |
| name: run with single worker thread and deadlock detection | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| - name: Cache Cargo home | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock | |
| - name: Run foyer-bench with single worker thread and deadlock detection | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: "--cfg tokio_unstable" | |
| RUST_LOG: info | |
| TOKIO_WORKER_THREADS: 1 | |
| CI: true | |
| run: |- | |
| cargo build --all --features deadlock | |
| mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage/deadlock | |
| timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60 | |
| asan: | |
| name: run with address saniziter | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }} | |
| - name: Cache Cargo home | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan | |
| - name: Run Unit Tests With Address Sanitizer | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable" | |
| RUST_LOG: info | |
| CI: true | |
| run: |- | |
| cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture | |
| - name: Run foyer-bench With Address Sanitizer | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable" | |
| RUST_LOG: info | |
| CI: true | |
| run: |- | |
| cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu | |
| mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan | |
| timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60 | |
| - name: Prepare Artifacts on Failure | |
| if: ${{ failure() }} | |
| run: |- | |
| find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.asan.tgz --transform 's#.*/##' | |
| - name: Upload Artifacts on Failure | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ failure() }} | |
| with: | |
| name: artifacts.asan.tgz | |
| path: artifacts.asan.tgz | |
| lsan: | |
| name: run with leak saniziter | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }} | |
| - name: Cache Cargo home | |
| uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-lsan | |
| - name: Run Unit Tests With Leak Sanitizer | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable" | |
| RUST_LOG: info | |
| CI: true | |
| run: |- | |
| cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture | |
| - name: Run foyer-bench With Leak Sanitizer | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable" | |
| RUST_LOG: info | |
| CI: true | |
| run: |- | |
| cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu | |
| mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan | |
| timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60 | |
| - name: Prepare Artifacts on Failure | |
| if: ${{ failure() }} | |
| run: |- | |
| find ./target/x86_64-unknown-linux-gnu/debug/ -type f -executable -name 'foyer*' -print0 | xargs -0 tar czvf artifacts.lsan.tgz --transform 's#.*/##' | |
| - name: Upload Artifacts on Failure | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ failure() }} | |
| with: | |
| name: artifacts.lsan.tgz | |
| path: artifacts.lsan.tgz | |
| # deterministic-test: | |
| # name: run deterministic test | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Checkout | |
| # uses: actions/checkout@v4 | |
| # - name: Install rust toolchain | |
| # uses: dtolnay/rust-toolchain@master | |
| # with: | |
| # toolchain: ${{ env.RUST_TOOLCHAIN }} | |
| # components: rustfmt, clippy | |
| # - name: Cache Cargo home | |
| # uses: actions/cache@v4 | |
| # id: cache | |
| # with: | |
| # path: | | |
| # ~/.cargo/bin/ | |
| # ~/.cargo/registry/index/ | |
| # ~/.cargo/registry/cache/ | |
| # ~/.cargo/git/db/ | |
| # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test | |
| # - if: steps.cache.outputs.cache-hit != 'true' | |
| # uses: taiki-e/install-action@nextest | |
| # - name: Run rust clippy check (madsim) | |
| # env: | |
| # RUST_BACKTRACE: 1 | |
| # RUSTFLAGS: "--cfg tokio_unstable --cfg madsim" | |
| # RUST_LOG: info | |
| # TOKIO_WORKER_THREADS: 1 | |
| # CI: true | |
| # run: |- | |
| # cargo clippy --all-targets | |
| # - name: Run nextest (madsim) | |
| # env: | |
| # RUST_BACKTRACE: 1 | |
| # RUSTFLAGS: "--cfg tokio_unstable --cfg madsim" | |
| # RUST_LOG: info | |
| # TOKIO_WORKER_THREADS: 1 | |
| # run: |- | |
| # cargo nextest run --all |