diff --git a/config/config.go b/config/config.go index 1951784dd1d..046c930be93 100644 --- a/config/config.go +++ b/config/config.go @@ -36,6 +36,7 @@ type Config struct { Experimental Experiments Plugins Plugins Pinning Pinning + Import Import Internal Internal // experimental/unstable options } diff --git a/config/import.go b/config/import.go new file mode 100644 index 00000000000..10af4edfa4d --- /dev/null +++ b/config/import.go @@ -0,0 +1,17 @@ +package config + +const ( + DefaultCidVersion = 0 + DefaultUnixFSRawLeaves = false + DefaultUnixFSChunker = "size-262144" + DefaultHashFunction = "sha2-256" +) + +// Import configures the default options for ingesting data. This affects commands +// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'. +type Import struct { + CidVersion OptionalInteger + UnixFSRawLeaves Flag + UnixFSChunker OptionalString + HashFunction OptionalString +} diff --git a/config/profile.go b/config/profile.go index 068498715c5..c73b05af27c 100644 --- a/config/profile.go +++ b/config/profile.go @@ -204,6 +204,28 @@ fetching may be degraded. return nil }, }, + "legacy-cid-v0": { + Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`, + + Transform: func(c *Config) error { + c.Import.CidVersion = *NewOptionalInteger(0) + c.Import.UnixFSRawLeaves = False + c.Import.UnixFSChunker = *NewOptionalString("size-262144") + c.Import.HashFunction = *NewOptionalString("sha2-256") + return nil + }, + }, + "test-cid-v1": { + Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`, + + Transform: func(c *Config) error { + c.Import.CidVersion = *NewOptionalInteger(1) + c.Import.UnixFSRawLeaves = True + c.Import.UnixFSChunker = *NewOptionalString("size-1048576") + c.Import.HashFunction = *NewOptionalString("sha2-256") + return nil + }, + }, } func getAvailablePort() (port int, err error) { diff --git a/core/commands/add.go b/core/commands/add.go index 4e59cc86758..94a5a0f51f5 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -8,6 +8,7 @@ import ( gopath "path" "strings" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/cheggaaa/pb" @@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information. cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."), cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), - cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"), + cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"), cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."), cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"), cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."), - cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"), + cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"), cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"), cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32), cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true), @@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information. return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + progress, _ := req.Options[progressOptionName].(bool) trickle, _ := req.Options[trickleOptionName].(bool) wrap, _ := req.Options[wrapOptionName].(bool) @@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information. inlineLimit, _ := req.Options[inlineLimitOptionName].(int) toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string) + if chunker == "" { + chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) + } + + if hashFunStr == "" { + hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) + } + + if !cidVerSet && !cfg.Import.CidVersion.IsDefault() { + cidVerSet = true + cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion)) + } + + if !rbset && cfg.Import.UnixFSRawLeaves != config.Default { + rbset = true + rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) + } + if onlyHash && toFilesSet { return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName) } diff --git a/core/commands/block.go b/core/commands/block.go index 6ceb258f62c..b4b0fd20457 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/boxo/files" + "github.com/ipfs/kubo/config" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" @@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0). }, Options: []cmds.Option{ cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"), - cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"), + cmds.StringOption(mhtypeOptionName, "Multihash hash function"), cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1), cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false), cmdutils.AllowBigBlockOption, @@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0). return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + mhtype, _ := req.Options[mhtypeOptionName].(string) + if mhtype == "" { + mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) + } + mhtval, ok := mh.Names[mhtype] if !ok { return fmt.Errorf("unrecognized multihash function: %s", mhtype) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 56aae4105da..ce5edb64167 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -87,7 +87,7 @@ into an object of the specified format. cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"), cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), - cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), + cmds.StringOption("hash", "Hash function to use"), cmdutils.AllowBigBlockOption, }, Run: dagPut, diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index c9c0b455b07..fb719916cdf 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -7,6 +7,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipldlegacy "github.com/ipfs/go-ipld-legacy" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipld/go-ipld-prime/multicodec" @@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + inputCodec, _ := req.Options["input-codec"].(string) storeCodec, _ := req.Options["store-codec"].(string) hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) + if hash == "" { + hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction) + } + var icodec mc.Code if err := icodec.Set(inputCodec); err != nil { return err diff --git a/core/commands/files.go b/core/commands/files.go index 9a7ee639a2c..12891a7301d 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -11,6 +11,7 @@ import ( "strings" humanize "github.com/dustin/go-humanize" + "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/core/commands/cmdenv" @@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information. return err } + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := nd.Repo.Config() + if err != nil { + return err + } + create, _ := req.Options[filesCreateOptionName].(bool) mkParents, _ := req.Options[filesParentsOptionName].(bool) trunc, _ := req.Options[filesTruncateOptionName].(bool) flush, _ := req.Options[filesFlushOptionName].(bool) rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool) - prefix, err := getPrefixNew(req) - if err != nil { - return err + if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default { + rawLeavesDef = true + rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves) } - nd, err := cmdenv.GetNode(env) + prefix, err := getPrefixNew(req) if err != nil { return err } diff --git a/docs/changelogs/v0.29.md b/docs/changelogs/v0.29.md index 386e93eadde..632c47b15bb 100644 --- a/docs/changelogs/v0.29.md +++ b/docs/changelogs/v0.29.md @@ -7,6 +7,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [Add search functionality for pin names](#add-search-functionality-for-pin-names) + - [Customizing `ipfs add` defaults](#customizing-ipfs-add-defaults) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -18,6 +19,16 @@ It is now possible to search for pins by name. To do so, use `ipfs pin ls --name "SomeName"`. The search is case-sensitive and will return all pins having a name which contains the exact word provided. +#### Customizing `ipfs add` defaults + +This release supports overriding global data ingestion defaults used by commands like `ipfs add` via user-defined [`Import.*` configuration options](../config.md#import). +The hash function, CID version, or UnixFS raw leaves and chunker behaviors can be set once, and used as the new implicit default for `ipfs add`. + +> [!TIP] +> As a convenience, two CID [profiles](../config.md#profile) are provided: `legacy-cid-v0` and `test-cid-v1`. +> A test profile that defaults to modern CIDv1 can be applied via `ipfs config profile apply test-cid-v1`. +> We encourage users to try it and report any issues. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index e36ffebb73e..130f724d5d2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -175,6 +175,11 @@ config file at runtime. - [`DNS`](#dns) - [`DNS.Resolvers`](#dnsresolvers) - [`DNS.MaxCacheTTL`](#dnsmaxcachettl) + - [`Import`](#import) + - [`Import.CidVersion`](#importcidversion) + - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves) + - [`Import.UnixFSChunker`](#importunixfschunker) + - [`Import.HashFunction`](#importhashfunction) ## Profiles @@ -265,6 +270,21 @@ documented in `ipfs config profile --help`. Use this profile with caution. +- `legacy-cid-v0` + + Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. + + > [!WARNING] + > This profile is provided for legacy users and should not be used for new projects. + +- `test-cid-v1` + + Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks. + + > [!NOTE] + > This profile will become the new implicit default, provided for testing purposes. + > Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details. + ## Types This document refers to the standard JSON types (e.g., `null`, `string`, @@ -2377,3 +2397,41 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s Default: Respect DNS Response TTL Type: `optionalDuration` + +## `Import` + +Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. + +Note that using flags will override the options defined here. + +### `Import.CidVersion` + +The default CID version. Commands affected: `ipfs add`. + +Default: `0` + +Type: `optionalInteger` + +### `Import.UnixFSRawLeaves` + +The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`. + +Default: `false` if `CidVersion=0`; `true` if `CidVersion=1` + +Type: `flag` + +### `Import.UnixFSChunker` + +The default UnixFS chunker. Commands affected: `ipfs add`. + +Default: `size-262144` + +Type: `optionalString` + +### `Import.HashFunction` + +The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`. + +Default: `sha2-256` + +Type: `optionalString` diff --git a/test/cli/add_test.go b/test/cli/add_test.go new file mode 100644 index 00000000000..ae652989ab5 --- /dev/null +++ b/test/cli/add_test.go @@ -0,0 +1,118 @@ +package cli + +import ( + "testing" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/require" +) + +func TestAdd(t *testing.T) { + t.Parallel() + + var ( + shortString = "hello world" + shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256 + shortStringCidV1 = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" // cidv1 - raw - sha2-256 + shortStringCidV1NoRawLeaves = "bafybeihykld7uyxzogax6vgyvag42y7464eywpf55gxi5qpoisibh3c5wa" // cidv1 - dag-pb - sha2-256 + shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6" + ) + + t.Run("produced cid version: implicit default (CIDv0)", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV0, cidStr) + }) + + t.Run("produced cid version: follows user-set configuration Import.CidVersion=0", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(0) + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV0, cidStr) + }) + + t.Run("produced cid multihash: follows user-set configuration in Import.HashFunction", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.HashFunction = *config.NewOptionalString("sha2-512") + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV1Sha512, cidStr) + }) + + t.Run("produced cid version: follows user-set configuration Import.CidVersion=1", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV1, cidStr) + }) + + t.Run("produced cid version: command flag overrides configuration in Import.CidVersion", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString, "--cid-version", "0") + require.Equal(t, shortStringCidV0, cidStr) + }) + + t.Run("produced unixfs raw leaves: follows user-set configuration Import.UnixFSRawLeaves", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + // CIDv1 defaults to raw-leaves=true + cfg.Import.CidVersion = *config.NewOptionalInteger(1) + // disable manually + cfg.Import.UnixFSRawLeaves = config.False + }) + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV1NoRawLeaves, cidStr) + }) + + t.Run("ipfs init --profile=legacy-cid-v0 sets config that produces legacy CIDv0", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0") + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV0, cidStr) + }) + + t.Run("ipfs init --profile=test-cid-v1 produces modern CIDv1", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init("--profile=test-cid-v1") + node.StartDaemon() + defer node.StopDaemon() + + cidStr := node.IPFSAddStr(shortString) + require.Equal(t, shortStringCidV1, cidStr) + }) +}