From a3374d99028d96a1ef262b81acb385690eb36f97 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Thu, 21 Apr 2022 12:41:58 -0300 Subject: [PATCH] refactor(block): CIDv1 and BlockPutSettings CidPrefix (#80) * feat(block options): add store codec * refactor: BlockPutSettings.CidPrefix Removes duplicated fields and replaces them with cid.Prefix Codec, MhType and MhLength were already in prefix, and we already return prefix. A lot of duplicated values and code responsible for syncing them did not really need to exist. * test: CIDv1 raw and dag-pb cases * chore: release 0.7.0 Co-authored-by: Marcin Rataj --- go.mod | 1 + go.sum | 2 + options/block.go | 134 ++++++++++++++++++++++++++++++----------------- tests/block.go | 103 +++++++++++++++++++++++++++++++++--- version.json | 2 +- 5 files changed, 185 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index e2c2a3a..b7fc215 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/libp2p/go-libp2p-core v0.8.5 github.com/multiformats/go-multiaddr v0.3.3 github.com/multiformats/go-multibase v0.0.3 + github.com/multiformats/go-multicodec v0.3.0 github.com/multiformats/go-multihash v0.0.15 ) diff --git a/go.sum b/go.sum index e1a94fd..c7e8297 100644 --- a/go.sum +++ b/go.sum @@ -713,6 +713,8 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= +github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= diff --git a/options/block.go b/options/block.go index 043dfde..1306486 100644 --- a/options/block.go +++ b/options/block.go @@ -2,15 +2,15 @@ package options import ( "fmt" + cid "github.com/ipfs/go-cid" + mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) type BlockPutSettings struct { - Codec string - MhType uint64 - MhLength int - Pin bool + CidPrefix cid.Prefix + Pin bool } type BlockRmSettings struct { @@ -20,53 +20,29 @@ type BlockRmSettings struct { type BlockPutOption func(*BlockPutSettings) error type BlockRmOption func(*BlockRmSettings) error -func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, cid.Prefix, error) { +func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) { + var cidPrefix cid.Prefix + + // Baseline is CIDv1 raw sha2-255-32 (can be tweaked later via opts) + cidPrefix.Version = 1 + cidPrefix.Codec = uint64(mc.Raw) + cidPrefix.MhType = mh.SHA2_256 + cidPrefix.MhLength = -1 // -1 means len is to be calculated during mh.Sum() + options := &BlockPutSettings{ - Codec: "", - MhType: mh.SHA2_256, - MhLength: -1, - Pin: false, + CidPrefix: cidPrefix, + Pin: false, } + // Apply any overrides for _, opt := range opts { err := opt(options) if err != nil { - return nil, cid.Prefix{}, err - } - } - - var pref cid.Prefix - pref.Version = 1 - - if options.Codec == "" { - if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) { - options.Codec = "protobuf" - } else { - options.Codec = "v0" - } - } - - if options.Codec == "v0" && options.MhType == mh.SHA2_256 { - pref.Version = 0 - } - - formatval, ok := cid.Codecs[options.Codec] - if !ok { - return nil, cid.Prefix{}, fmt.Errorf("unrecognized format: %s", options.Codec) - } - - if options.Codec == "v0" { - if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) { - return nil, cid.Prefix{}, fmt.Errorf("only sha2-255-32 is allowed with CIDv0") + return nil, err } } - pref.Codec = formatval - - pref.MhType = options.MhType - pref.MhLength = options.MhLength - - return options, pref, nil + return options, nil } func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) { @@ -87,13 +63,75 @@ type blockOpts struct{} var Block blockOpts -// Format is an option for Block.Put which specifies the multicodec to use to -// serialize the object. Default is "v0" -func (blockOpts) Format(codec string) BlockPutOption { +// CidCodec is the modern option for Block.Put which specifies the multicodec to use +// in the CID returned by the Block.Put operation. +// It uses correct codes from go-multicodec and replaces the old Format now with CIDv1 as the default. +func (blockOpts) CidCodec(codecName string) BlockPutOption { + return func(settings *BlockPutSettings) error { + if codecName == "" { + return nil + } + code, err := codeFromName(codecName) + if err != nil { + return err + } + settings.CidPrefix.Codec = uint64(code) + return nil + } +} + +// Map string to code from go-multicodec +func codeFromName(codecName string) (mc.Code, error) { + var cidCodec mc.Code + err := cidCodec.Set(codecName) + return cidCodec, err +} + +// Format is a legacy option for Block.Put which specifies the multicodec to +// use to serialize the object. +// Provided for backward-compatibility only. Use CidCodec instead. +func (blockOpts) Format(format string) BlockPutOption { return func(settings *BlockPutSettings) error { - settings.Codec = codec + if format == "" { + return nil + } + // Opt-in CIDv0 support for backward-compatibility + if format == "v0" { + settings.CidPrefix.Version = 0 + } + + // Fixup a legacy (invalid) names for dag-pb (0x70) + if format == "v0" || format == "protobuf" { + format = "dag-pb" + } + + // Fixup invalid name for dag-cbor (0x71) + if format == "cbor" { + format = "dag-cbor" + } + + // Set code based on name passed as "format" + code, err := codeFromName(format) + if err != nil { + return err + } + settings.CidPrefix.Codec = uint64(code) + + // If CIDv0, ensure all parameters are compatible + // (in theory go-cid would validate this anyway, but we want to provide better errors) + pref := settings.CidPrefix + if pref.Version == 0 { + if pref.Codec != uint64(mc.DagPb) { + return fmt.Errorf("only dag-pb is allowed with CIDv0") + } + if pref.MhType != mh.SHA2_256 || (pref.MhLength != -1 && pref.MhLength != 32) { + return fmt.Errorf("only sha2-255-32 is allowed with CIDv0") + } + } + return nil } + } // Hash is an option for Block.Put which specifies the multihash settings to use @@ -101,8 +139,8 @@ func (blockOpts) Format(codec string) BlockPutOption { // If mhLen is set to -1, default length for the hash will be used func (blockOpts) Hash(mhType uint64, mhLen int) BlockPutOption { return func(settings *BlockPutSettings) error { - settings.MhType = mhType - settings.MhLength = mhLen + settings.CidPrefix.MhType = mhType + settings.CidPrefix.MhLength = mhLen return nil } } diff --git a/tests/block.go b/tests/block.go index 87fa90b..916e52d 100644 --- a/tests/block.go +++ b/tests/block.go @@ -17,15 +17,19 @@ import ( ) var ( - pbCid = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" - cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" - cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" + pbCidV0 = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" // dag-pb + pbCid = "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // dag-pb + rawCid = "bafkreiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // raw bytes + cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" // dag-cbor + cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" // dag-cbor keccak-512 ) +// dag-pb func pbBlock() io.Reader { return bytes.NewReader([]byte{10, 12, 8, 2, 18, 6, 104, 101, 108, 108, 111, 10, 24, 6}) } +// dag-cbor func cborBlock() io.Reader { return bytes.NewReader([]byte{101, 72, 101, 108, 108, 111}) } @@ -38,8 +42,12 @@ func (tp *TestSuite) TestBlock(t *testing.T) { return nil }) - t.Run("TestBlockPut", tp.TestBlockPut) - t.Run("TestBlockPutFormat", tp.TestBlockPutFormat) + t.Run("TestBlockPut (get raw CIDv1)", tp.TestBlockPut) + t.Run("TestBlockPutCidCodec: dag-pb", tp.TestBlockPutCidCodecDagPb) + t.Run("TestBlockPutCidCodec: dag-cbor", tp.TestBlockPutCidCodecDagCbor) + t.Run("TestBlockPutFormat (legacy): cbor → dag-cbor", tp.TestBlockPutFormatDagCbor) + t.Run("TestBlockPutFormat (legacy): protobuf → dag-pb", tp.TestBlockPutFormatDagPb) + t.Run("TestBlockPutFormat (legacy): v0 → CIDv0", tp.TestBlockPutFormatV0) t.Run("TestBlockPutHash", tp.TestBlockPutHash) t.Run("TestBlockGet", tp.TestBlockGet) t.Run("TestBlockRm", tp.TestBlockRm) @@ -47,6 +55,7 @@ func (tp *TestSuite) TestBlock(t *testing.T) { t.Run("TestBlockPin", tp.TestBlockPin) } +// when no opts are passed, produced CID has 'raw' codec func (tp *TestSuite) TestBlockPut(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -60,12 +69,14 @@ func (tp *TestSuite) TestBlockPut(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != pbCid { + if res.Path().Cid().String() != rawCid { t.Errorf("got wrong cid: %s", res.Path().Cid().String()) } } -func (tp *TestSuite) TestBlockPutFormat(t *testing.T) { +// Format is deprecated, it used invalid codec names. +// Confirm 'cbor' gets fixed to 'dag-cbor' +func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() api, err := tp.makeAPI(ctx) @@ -83,6 +94,82 @@ func (tp *TestSuite) TestBlockPutFormat(t *testing.T) { } } +// Format is deprecated, it used invalid codec names. +// Confirm 'protobuf' got fixed to 'dag-pb' +func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("protobuf")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +// Format is deprecated, it used invalid codec names. +// Confirm fake codec 'v0' got fixed to CIDv0 (with implicit dag-pb codec) +func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("v0")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCidV0 { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, cborBlock(), opt.Block.CidCodec("dag-cbor")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + +func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + api, err := tp.makeAPI(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := api.Block().Put(ctx, pbBlock(), opt.Block.CidCodec("dag-pb")) + if err != nil { + t.Fatal(err) + } + + if res.Path().Cid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + } +} + func (tp *TestSuite) TestBlockPutHash(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -95,7 +182,7 @@ func (tp *TestSuite) TestBlockPutHash(t *testing.T) { ctx, cborBlock(), opt.Block.Hash(mh.KECCAK_512, -1), - opt.Block.Format("cbor"), + opt.Block.CidCodec("dag-cbor"), ) if err != nil { t.Fatal(err) diff --git a/version.json b/version.json index b069fc6..9f6d6fc 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "version": "v0.6.2" + "version": "v0.7.0" }