allow storage miners to manage a blocklist of piece CIDs

api/api_storage.go

@@ -56,6 +56,8 @@ type StorageMiner interface {
 	DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error
 	DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error)
 	DealsSetAcceptingStorageDeals(context.Context, bool) error
+	DealsPieceCidBlacklist(context.Context) ([]cid.Cid, error)
+	DealsSetPieceCidBlacklist(context.Context, []cid.Cid) error
 
 	StorageAddLocal(ctx context.Context, path string) error
 }

api/apistruct/struct.go

@@ -226,6 +226,8 @@ type StorageMinerStruct struct {
 		DealsImportData               func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"`
 		DealsList                     func(ctx context.Context) ([]storagemarket.StorageDeal, error)    `perm:"read"`
 		DealsSetAcceptingStorageDeals func(context.Context, bool) error                                 `perm:"admin"`
+		DealsPieceCidBlacklist        func(context.Context) ([]cid.Cid, error)                          `perm:"admin"`
+		DealsSetPieceCidBlacklist     func(context.Context, []cid.Cid) error                            `perm:"read"`
 
 		StorageAddLocal func(ctx context.Context, path string) error `perm:"admin"`
 	}
@@ -872,6 +874,14 @@ func (c *StorageMinerStruct) DealsSetAcceptingStorageDeals(ctx context.Context,
 	return c.Internal.DealsSetAcceptingStorageDeals(ctx, b)
 }
 
+func (c *StorageMinerStruct) DealsPieceCidBlacklist(ctx context.Context) ([]cid.Cid, error) {
+	return c.Internal.DealsPieceCidBlacklist(ctx)
+}
+
+func (c *StorageMinerStruct) DealsSetPieceCidBlacklist(ctx context.Context, cids []cid.Cid) error {
+	return c.Internal.DealsSetPieceCidBlacklist(ctx, cids)
+}
+
 func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) error {
 	return c.Internal.StorageAddLocal(ctx, path)
 }

cmd/lotus-storage-miner/market.go

@@ -1,14 +1,18 @@
 package main
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"text/tabwriter"
 	"time"
 
 	"github.com/docker/go-units"
 	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cidutil/cidenc"
+	"github.com/multiformats/go-multibase"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
@@ -20,6 +24,32 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 )
 
+var CidBaseFlag = cli.StringFlag{
+	Name:        "cid-base",
+	Hidden:      true,
+	Value:       "base32",
+	Usage:       "Multibase encoding used for version 1 CIDs in output.",
+	DefaultText: "base32",
+}
+
+// GetCidEncoder returns an encoder using the `cid-base` flag if provided, or
+// the default (Base32) encoder if not.
+func GetCidEncoder(cctx *cli.Context) (cidenc.Encoder, error) {
+	val := cctx.String("cid-base")
+
+	e := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)}
+
+	if val != "" {
+		var err error
+		e.Base, err = multibase.EncoderByName(val)
+		if err != nil {
+			return e, err
+		}
+	}
+
+	return e, nil
+}
+
 var enableCmd = &cli.Command{
 	Name:  "enable",
 	Usage: "Configure the miner to consider storage deal proposals",
@@ -197,6 +227,9 @@ var dealsCmd = &cli.Command{
 		disableCmd,
 		setAskCmd,
 		getAskCmd,
+		setBlacklistCmd,
+		getBlacklistCmd,
+		resetBlacklistCmd,
 	},
 }
 
@@ -255,3 +288,96 @@ var dealsListCmd = &cli.Command{
 		return nil
 	},
 }
+
+var getBlacklistCmd = &cli.Command{
+	Name:  "get-blacklist",
+	Usage: "List the contents of the storage miner's piece CID blacklist",
+	Flags: []cli.Flag{
+		&CidBaseFlag,
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		blacklist, err := api.DealsPieceCidBlacklist(lcli.DaemonContext(cctx))
+		if err != nil {
+			return err
+		}
+
+		encoder, err := GetCidEncoder(cctx)
+		if err != nil {
+			return err
+		}
+
+		for idx := range blacklist {
+			fmt.Println(encoder.Encode(blacklist[idx]))
+		}
+
+		return nil
+	},
+}
+
+var setBlacklistCmd = &cli.Command{
+	Name:      "set-blacklist",
+	Usage:     "Set the storage miner's list of blacklisted piece CIDs",
+	ArgsUsage: "[<path-of-file-containing-newline-delimited-piece-CIDs> (optional, will read from stdin if omitted)]",
+	Flags:     []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		scanner := bufio.NewScanner(os.Stdin)
+		if cctx.Args().Present() && cctx.Args().First() != "-" {
+			absPath, err := filepath.Abs(cctx.Args().First())
+			if err != nil {
+				return err
+			}
+
+			file, err := os.Open(absPath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			defer file.Close()
+
+			scanner = bufio.NewScanner(file)
+		}
+
+		var blacklist []cid.Cid
+		for scanner.Scan() {
+			decoded, err := cid.Decode(scanner.Text())
+			if err != nil {
+				return err
+			}
+
+			blacklist = append(blacklist, decoded)
+		}
+
+		err = scanner.Err()
+		if err != nil {
+			return err
+		}
+
+		return api.DealsSetPieceCidBlacklist(lcli.DaemonContext(cctx), blacklist)
+	},
+}
+
+var resetBlacklistCmd = &cli.Command{
+	Name:  "reset-blacklist",
+	Usage: "Remove all entries from the storage miner's piece CID blacklist",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		return api.DealsSetPieceCidBlacklist(lcli.DaemonContext(cctx), []cid.Cid{})
+	},
+}

node/builder.go

@@ -314,6 +314,8 @@ func Online() Option {
 
 			Override(new(dtypes.AcceptingStorageDealsConfigFunc), modules.NewAcceptingStorageDealsConfigFunc),
 			Override(new(dtypes.SetAcceptingStorageDealsConfigFunc), modules.NewSetAcceptingStorageDealsConfigFunc),
+			Override(new(dtypes.StorageDealPieceCidBlacklistConfigFunc), modules.NewStorageDealPieceCidBlacklistConfigFunc),
+			Override(new(dtypes.SetStorageDealPieceCidBlacklistConfigFunc), modules.NewSetStorageDealPieceCidBlacklistConfigFunc),
 		),
 	)
 }

node/config/def.go

@@ -4,6 +4,8 @@ import (
 	"encoding"
 	"time"
 
+	"github.com/ipfs/go-cid"
+
 	sectorstorage "github.com/filecoin-project/sector-storage"
 )
 
@@ -33,6 +35,7 @@ type StorageMiner struct {
 
 type DealmakingConfig struct {
 	AcceptingStorageDeals bool
+	PieceCidBlacklist     []cid.Cid
 }
 
 // API contains configs for API endpoint
@@ -121,6 +124,7 @@ func DefaultStorageMiner() *StorageMiner {
 
 		Dealmaking: DealmakingConfig{
 			AcceptingStorageDeals: true,
+			PieceCidBlacklist:     []cid.Cid{},
 		},
 	}
 	cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"

node/impl/storminer.go

@@ -43,7 +43,9 @@ type StorageMinerAPI struct {
 	StorageMgr      *sectorstorage.Manager `optional:"true"`
 	*stores.Index
 
-	SetAcceptingStorageDealsConfigFunc dtypes.SetAcceptingStorageDealsConfigFunc
+	SetAcceptingStorageDealsConfigFunc        dtypes.SetAcceptingStorageDealsConfigFunc
+	StorageDealPieceCidBlacklistConfigFunc    dtypes.StorageDealPieceCidBlacklistConfigFunc
+	SetStorageDealPieceCidBlacklistConfigFunc dtypes.SetStorageDealPieceCidBlacklistConfigFunc
 }
 
 func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
@@ -232,6 +234,14 @@ func (sm *StorageMinerAPI) DealsImportData(ctx context.Context, deal cid.Cid, fn
 	return sm.StorageProvider.ImportDataForDeal(ctx, deal, fi)
 }
 
+func (sm *StorageMinerAPI) DealsPieceCidBlacklist(ctx context.Context) ([]cid.Cid, error) {
+	return sm.StorageDealPieceCidBlacklistConfigFunc()
+}
+
+func (sm *StorageMinerAPI) DealsSetPieceCidBlacklist(ctx context.Context, cids []cid.Cid) error {
+	return sm.SetStorageDealPieceCidBlacklistConfigFunc(cids)
+}
+
 func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) error {
 	if sm.StorageMgr == nil {
 		return xerrors.Errorf("no storage manager")

node/modules/dtypes/miner.go

@@ -1,6 +1,8 @@
 package dtypes
 
 import (
+	"github.com/ipfs/go-cid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 )
@@ -15,3 +17,12 @@ type AcceptingStorageDealsConfigFunc func() (bool, error)
 // SetAcceptingStorageDealsFunc is a function which is used to disable or enable
 // storage deal acceptance.
 type SetAcceptingStorageDealsConfigFunc func(bool) error
+
+// StorageDealCidBlacklistConfigFunc is a function which reads from miner config
+// to obtain a list of CIDs for which the storage miner will not accept storage
+// proposals.
+type StorageDealPieceCidBlacklistConfigFunc func() ([]cid.Cid, error)
+
+// SetStorageDealCidBlacklistConfigFunc is a function which is used to set a
+// list of CIDs for which the storage miner will reject deal proposals.
+type SetStorageDealPieceCidBlacklistConfigFunc func([]cid.Cid) error