From b386bdef847382d14b2867023093e3926fc41b94 Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Fri, 27 Sep 2024 18:13:49 +0800 Subject: [PATCH] smoke: add smoking test for chunk dedup Add smoking test case for chunk dedup. Signed-off-by: Yadong Ding --- smoke/go.mod | 2 + smoke/go.sum | 6 ++ smoke/tests/chunk_dedup_test.go | 97 +++++++++++++++++++++++++++++++++ smoke/tests/tool/context.go | 1 + smoke/tests/tool/nydusd.go | 5 ++ storage/src/cache/cachedfile.rs | 13 ++++- 6 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 smoke/tests/chunk_dedup_test.go diff --git a/smoke/go.mod b/smoke/go.mod index dbf95cf434f..d68c7193fba 100644 --- a/smoke/go.mod +++ b/smoke/go.mod @@ -12,6 +12,7 @@ require ( github.com/pkg/xattr v0.4.9 github.com/stretchr/testify v1.8.4 golang.org/x/sys v0.15.0 + github.com/mattn/go-sqlite3 v1.14.23 ) require ( @@ -27,6 +28,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/mattn/go-sqlite3 v1.14.23 // indirect github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect diff --git a/smoke/go.sum b/smoke/go.sum index 0fa56cfafd8..67e97a62e87 100644 --- a/smoke/go.sum +++ b/smoke/go.sum @@ -10,6 +10,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= +github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= @@ -53,6 +54,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -67,7 +69,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -135,6 +140,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/smoke/tests/chunk_dedup_test.go b/smoke/tests/chunk_dedup_test.go new file mode 100644 index 00000000000..97d3365f7c2 --- /dev/null +++ b/smoke/tests/chunk_dedup_test.go @@ -0,0 +1,97 @@ +// Copyright 2024 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "database/sql" + "fmt" + "path/filepath" + "testing" + + _ "github.com/mattn/go-sqlite3" + + "github.com/containerd/nydus-snapshotter/pkg/converter" + "github.com/dragonflyoss/nydus/smoke/tests/texture" + "github.com/dragonflyoss/nydus/smoke/tests/tool" + "github.com/dragonflyoss/nydus/smoke/tests/tool/test" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +type ChunkDedupTestSuite struct{} + +func (c *ChunkDedupTestSuite) TestChunkDedup() test.Generator { + + scenarios := tool.DescartesIterator{} + scenarios.Dimension(paramEnablePrefetch, []interface{}{false, true}) + + return func() (name string, testCase test.Case) { + if !scenarios.HasNext() { + return + } + scenario := scenarios.Next() + + return scenario.Str(), func(t *testing.T) { + c.testCasTables(t, scenario.GetBool(paramEnablePrefetch)) + } + } +} + +func (c *ChunkDedupTestSuite) testCasTables(t *testing.T, enablePrefetch bool) { + ctx := tool.DefaultContext(t) + + // Prepare work directory + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + + lowerLayer := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "source")) + lowerOCIBlobDigest, lowerRafsBlobDigest := lowerLayer.PackRef(t, *ctx, ctx.Env.BlobDir, ctx.Build.OCIRefGzip) + mergeOption := converter.MergeOption{ + BuilderPath: ctx.Binary.Builder, + ChunkDictPath: "", + OCIRef: true, + } + actualDigests, lowerBootstrap := tool.MergeLayers(t, *ctx, mergeOption, []converter.Layer{ + { + Digest: lowerRafsBlobDigest, + OriginalDigest: &lowerOCIBlobDigest, + }, + }) + require.Equal(t, []digest.Digest{lowerOCIBlobDigest}, actualDigests) + + ctx.Env.BootstrapPath = lowerBootstrap + ctx.Runtime.EnablePrefetch = enablePrefetch + ctx.Runtime.ChunkDedupDb = filepath.Join(ctx.Env.WorkDir, "cas.db") + + nydusd, err := tool.NewNydusdWithContext(*ctx) + require.NoError(t, err) + err = nydusd.Mount() + require.NoError(t, err) + defer nydusd.Umount() + nydusd.Verify(t, lowerLayer.FileTree) + + db, err := sql.Open("sqlite3", ctx.Runtime.ChunkDedupDb) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + for _, expectedTable := range []string{"Blobs", "Chunks"} { + var count int + query := fmt.Sprintf("SELECT COUNT(*) FROM %s;", expectedTable) + err := db.QueryRow(query).Scan(&count) + require.NoError(t, err) + if expectedTable == "Blobs" { + require.Equal(t, count, 1) + } else { + require.Equal(t, count, 8) + } + } + +} + +func TestChunkDedup(t *testing.T) { + test.Run(t, &ChunkDedupTestSuite{}) +} diff --git a/smoke/tests/tool/context.go b/smoke/tests/tool/context.go index 04cf6d851cf..b1215f59b7d 100644 --- a/smoke/tests/tool/context.go +++ b/smoke/tests/tool/context.go @@ -39,6 +39,7 @@ type RuntimeContext struct { RafsMode string EnablePrefetch bool AmplifyIO uint64 + ChunkDedupDb string } type EnvContext struct { diff --git a/smoke/tests/tool/nydusd.go b/smoke/tests/tool/nydusd.go index c340cce88b1..6f1915eddee 100644 --- a/smoke/tests/tool/nydusd.go +++ b/smoke/tests/tool/nydusd.go @@ -74,6 +74,7 @@ type NydusdConfig struct { AccessPattern bool PrefetchFiles []string AmplifyIO uint64 + ChunkDedupDb string // Hot Upgrade config. Upgrade bool SupervisorSockPath string @@ -193,6 +194,9 @@ func newNydusd(conf NydusdConfig) (*Nydusd, error) { if len(conf.BootstrapPath) > 0 { args = append(args, "--bootstrap", conf.BootstrapPath) } + if len(conf.ChunkDedupDb) > 0 { + args = append(args, "--dedup-db", conf.ChunkDedupDb) + } if conf.Upgrade { args = append(args, "--upgrade") } @@ -276,6 +280,7 @@ func NewNydusdWithContext(ctx Context) (*Nydusd, error) { RafsMode: ctx.Runtime.RafsMode, DigestValidate: false, AmplifyIO: ctx.Runtime.AmplifyIO, + ChunkDedupDb: ctx.Runtime.ChunkDedupDb, } if err := makeConfig(NydusdConfigTpl, conf); err != nil { diff --git a/storage/src/cache/cachedfile.rs b/storage/src/cache/cachedfile.rs index a2432cb5fa7..43c65cc3f77 100644 --- a/storage/src/cache/cachedfile.rs +++ b/storage/src/cache/cachedfile.rs @@ -299,7 +299,10 @@ impl FileCacheEntry { Self::_update_chunk_pending_status(&delayed_chunk_map, chunk.as_ref(), res.is_ok()); if let Some(mgr) = cas_mgr { if let Err(e) = mgr.record_chunk(&blob_info, chunk.deref(), file_path.as_ref()) { - warn!("failed to record chunk state for dedup, {}", e); + warn!( + "failed to record chunk state for dedup in delay_persist_chunk_data, {}", + e + ); } } }); @@ -309,6 +312,14 @@ impl FileCacheEntry { let offset = chunk.uncompressed_offset(); let res = Self::persist_cached_data(&self.file, offset, buf); self.update_chunk_pending_status(chunk, res.is_ok()); + if let Some(mgr) = &self.cas_mgr { + if let Err(e) = mgr.record_chunk(&self.blob_info, chunk, self.file_path.as_ref()) { + warn!( + "failed to record chunk state for dedup in persist_chunk_data, {}", + e + ); + } + } } fn persist_cached_data(file: &Arc, offset: u64, buffer: &[u8]) -> Result<()> {