diff --git a/.github/image/Dockerfile b/.github/image/Dockerfile
index 2c7981f34..7c808d9ef 100644
--- a/.github/image/Dockerfile
+++ b/.github/image/Dockerfile
@@ -18,7 +18,7 @@ RUN git submodule update --init
RUN go mod download
# Download Go dependencies for Curio
-WORKDIR /app/boost
+WORKDIR /app/curio
RUN git submodule update --init
RUN go mod download
diff --git a/.gitignore b/.gitignore
index ee9bf7cf3..9f2c5e3fc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
/booster-bitswap
/docgen-md
/docgen-openrpc
+/migrate-curio
extern/filecoin-ffi/rust/target
extern/boostd-data/boostd-data
**/*.a
diff --git a/Makefile b/Makefile
index 1e77c8f34..6968c7204 100644
--- a/Makefile
+++ b/Makefile
@@ -133,6 +133,11 @@ boostci: $(BUILD_DEPS)
$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
.PHONY: boostci
+migrate-curio: $(BUILD_DEPS)
+ rm -f migrate-curio
+ $(GOCC) build $(GOFLAGS) -o migrate-curio ./cmd/migrate-curio
+.PHONY: boostci
+
react: validate-node-version
npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
npm run --prefix react build
@@ -143,7 +148,7 @@ update-react: validate-node-version
npm run --prefix react build
.PHONY: react
-build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid migrate-curio
.PHONY: build-go
build: react build-go
@@ -162,6 +167,7 @@ install-boost:
install -C ./booster-http /usr/local/bin/booster-http
install -C ./booster-bitswap /usr/local/bin/booster-bitswap
install -C ./migrate-lid /usr/local/bin/migrate-lid
+ install -C ./migrate-curio /usr/local/bin/migrate-curio
install-devnet:
install -C ./devnet /usr/local/bin/devnet
diff --git a/api/api.go b/api/api.go
index 91f4fd425..6f7a0b943 100644
--- a/api/api.go
+++ b/api/api.go
@@ -26,6 +26,7 @@ type Boost interface {
Net
// MethodGroup: Boost
+ BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin
BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin
BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin
diff --git a/api/proxy_gen.go b/api/proxy_gen.go
index 8f3ce38a0..b98bf78bb 100644
--- a/api/proxy_gen.go
+++ b/api/proxy_gen.go
@@ -60,6 +60,8 @@ type BoostStruct struct {
BoostIndexerListMultihashes func(p0 context.Context, p1 []byte) ([]multihash.Multihash, error) `perm:"admin"`
+ BoostIndexerRemoveAll func(p0 context.Context) ([]cid.Cid, error) `perm:"admin"`
+
BoostLegacyDealByProposalCid func(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) `perm:"admin"`
BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string, p3 bool) (*ProviderDealRejectionInfo, error) `perm:"admin"`
@@ -355,6 +357,17 @@ func (s *BoostStub) BoostIndexerListMultihashes(p0 context.Context, p1 []byte) (
return *new([]multihash.Multihash), ErrNotSupported
}
+func (s *BoostStruct) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) {
+ if s.Internal.BoostIndexerRemoveAll == nil {
+ return *new([]cid.Cid), ErrNotSupported
+ }
+ return s.Internal.BoostIndexerRemoveAll(p0)
+}
+
+func (s *BoostStub) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) {
+ return *new([]cid.Cid), ErrNotSupported
+}
+
func (s *BoostStruct) BoostLegacyDealByProposalCid(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) {
if s.Internal.BoostLegacyDealByProposalCid == nil {
return *new(legacytypes.MinerDeal), ErrNotSupported
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
index b3922ced3..59679d547 100644
Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ
diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go
index 0e4b0fd69..35dc919dd 100644
--- a/cmd/boostd/index.go
+++ b/cmd/boostd/index.go
@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
indexProvAnnounceLatestHttp,
indexProvAnnounceDealRemovalAd,
indexProvAnnounceDeal,
+ indexProvRemoveAllCmd,
},
}
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
return nil
},
}
+
+var indexProvRemoveAllCmd = &cli.Command{
+ Name: "remove-all",
+ Usage: "Announce all removal ad for all contextIDs",
+ Flags: []cli.Flag{},
+ Action: func(cctx *cli.Context) error {
+ ctx := lcli.ReqContext(cctx)
+
+ // get boost api
+ napi, closer, err := bcli.GetBoostAPI(cctx)
+ if err != nil {
+ return err
+ }
+ defer closer()
+
+ // announce markets and boost deals
+ cids, err := napi.BoostIndexerRemoveAll(ctx)
+ if err != nil {
+ return err
+ }
+ for _, c := range cids {
+ fmt.Println("Published the removal ad with CID", c.String())
+ }
+ return nil
+ },
+}
diff --git a/cmd/migrate-curio/leveldb.go b/cmd/migrate-curio/leveldb.go
new file mode 100644
index 000000000..d4b7f8ebd
--- /dev/null
+++ b/cmd/migrate-curio/leveldb.go
@@ -0,0 +1,19 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/urfave/cli/v2"
+)
+
+var cleanupLevelDBCmd = &cli.Command{
+ Name: "leveldb",
+ Description: "Removes the indexes and other metadata leveldb based LID store",
+ Usage: "migrate-curio cleanup leveldb",
+ Before: before,
+ Action: func(cctx *cli.Context) error {
+ fmt.Println("Please remove the directory called 'LID' in the boost repo path to remove leveldb based LID")
+ fmt.Println("This directory can also be present outside of Boost repo if 'boostd-data' was running with a custom repo path")
+ return nil
+ },
+}
diff --git a/cmd/migrate-curio/main.go b/cmd/migrate-curio/main.go
new file mode 100644
index 000000000..3aa8399d6
--- /dev/null
+++ b/cmd/migrate-curio/main.go
@@ -0,0 +1,88 @@
+package main
+
+import (
+ "os"
+
+ "github.com/filecoin-project/boost/build"
+ logging "github.com/ipfs/go-log/v2"
+ "github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("migrate-curio")
+
+const (
+ FlagBoostRepo = "boost-repo"
+)
+
+var FlagRepo = &cli.StringFlag{
+ Name: FlagBoostRepo,
+ EnvVars: []string{"BOOST_PATH"},
+ Usage: "boost repo path",
+ Value: "~/.boost",
+}
+
+var IsVeryVerbose bool
+
+var FlagVeryVerbose = &cli.BoolFlag{
+ Name: "vv",
+ Usage: "enables very verbose mode, useful for debugging the CLI",
+ Destination: &IsVeryVerbose,
+}
+
+func main() {
+ app := &cli.App{
+ Name: "migrate-curio",
+ Usage: "Migrate boost to Curio",
+ EnableBashCompletion: true,
+ Version: build.UserVersion(),
+ Flags: []cli.Flag{
+ FlagRepo,
+ FlagVeryVerbose,
+ &cli.StringFlag{
+ Name: "db-host",
+ EnvVars: []string{"CURIO_DB_HOST", "CURIO_HARMONYDB_HOSTS"},
+ Usage: "Command separated list of hostnames for yugabyte cluster",
+ Value: "127.0.0.1",
+ },
+ &cli.StringFlag{
+ Name: "db-name",
+ EnvVars: []string{"CURIO_DB_NAME", "CURIO_HARMONYDB_NAME"},
+ Value: "yugabyte",
+ },
+ &cli.StringFlag{
+ Name: "db-user",
+ EnvVars: []string{"CURIO_DB_USER", "CURIO_HARMONYDB_USERNAME"},
+ Value: "yugabyte",
+ },
+ &cli.StringFlag{
+ Name: "db-password",
+ EnvVars: []string{"CURIO_DB_PASSWORD", "CURIO_HARMONYDB_PASSWORD"},
+ Value: "yugabyte",
+ },
+ &cli.StringFlag{
+ Name: "db-port",
+ EnvVars: []string{"CURIO_DB_PORT", "CURIO_HARMONYDB_PORT"},
+ Value: "5433",
+ },
+ },
+ Commands: []*cli.Command{
+ migrateCmd,
+ cleanupLIDCmd,
+ },
+ }
+ app.Setup()
+
+ if err := app.Run(os.Args); err != nil {
+ os.Stderr.WriteString("Error: " + err.Error() + "\n")
+ }
+}
+
+func before(cctx *cli.Context) error {
+ _ = logging.SetLogLevel("migrate-curio", "INFO")
+
+ if IsVeryVerbose {
+ _ = logging.SetLogLevel("migrate-curio", "DEBUG")
+ }
+
+ return nil
+}
diff --git a/cmd/migrate-curio/migrate.go b/cmd/migrate-curio/migrate.go
new file mode 100644
index 000000000..df87ada17
--- /dev/null
+++ b/cmd/migrate-curio/migrate.go
@@ -0,0 +1,623 @@
+package main
+
+import (
+ "bytes"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "path"
+ "time"
+
+ "github.com/filecoin-project/boost/db"
+ "github.com/filecoin-project/boost/lib/legacy"
+ "github.com/filecoin-project/boost/node/repo"
+ "github.com/filecoin-project/boost/storagemarket/types"
+ "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints"
+ "github.com/filecoin-project/boost/storagemarket/types/legacytypes"
+ transportTypes "github.com/filecoin-project/boost/transport/types"
+ "github.com/filecoin-project/curio/deps"
+ "github.com/filecoin-project/curio/harmony/harmonydb"
+ "github.com/filecoin-project/go-address"
+ "github.com/filecoin-project/go-bitfield"
+ vfsm "github.com/filecoin-project/go-ds-versioning/pkg/fsm"
+ "github.com/filecoin-project/go-state-types/abi"
+ verifreg9types "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
+ "github.com/filecoin-project/go-statemachine/fsm"
+ "github.com/filecoin-project/lotus/api/v1api"
+ "github.com/filecoin-project/lotus/blockstore"
+ "github.com/filecoin-project/lotus/chain/actors/adt"
+ "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+ ltypes "github.com/filecoin-project/lotus/chain/types"
+ lcli "github.com/filecoin-project/lotus/cli"
+ lotus_repo "github.com/filecoin-project/lotus/node/repo"
+ "github.com/ipfs/go-datastore"
+ "github.com/ipfs/go-datastore/namespace"
+ cbor "github.com/ipfs/go-ipld-cbor"
+ "github.com/mitchellh/go-homedir"
+ "github.com/urfave/cli/v2"
+ "golang.org/x/net/context"
+ "golang.org/x/xerrors"
+)
+
+var migrateCmd = &cli.Command{
+ Name: "migrate",
+ Description: "Migrate boost metadata to Curio",
+ Usage: "migrate-curio migrate",
+ Before: before,
+ Action: func(cctx *cli.Context) error {
+ repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo))
+ if err != nil {
+ return err
+ }
+
+ return migrate(cctx, repoDir)
+ },
+}
+
+func migrate(cctx *cli.Context, repoDir string) error {
+ ctx := cctx.Context
+
+ r, err := lotus_repo.NewFS(repoDir)
+ if err != nil {
+ return err
+ }
+ ok, err := r.Exists()
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return fmt.Errorf("repo at '%s' is not initialized", cctx.String(FlagBoostRepo))
+ }
+
+ lr, err := r.Lock(repo.Boost)
+ if err != nil {
+ return err
+ }
+
+ mds, err := lr.Datastore(ctx, "/metadata")
+ if err != nil {
+ return err
+ }
+
+ maddrb, err := mds.Get(context.TODO(), datastore.NewKey("miner-address"))
+ if err != nil {
+ return err
+ }
+
+ maddr, err := address.NewFromBytes(maddrb)
+ if err != nil {
+ return err
+ }
+
+ // Connect to full Node
+ full, closer, err := lcli.GetFullNodeAPIV1(cctx)
+ if err != nil {
+ return xerrors.Errorf("failed to connect to full node API: %w", err)
+ }
+ defer closer()
+
+ // Connect to Harmony DB
+ hdb, err := deps.MakeDB(cctx)
+ if err != nil {
+ return xerrors.Errorf("failed to connect to harmony DB: %w", err)
+ }
+
+ dbPath := path.Join(repoDir, "boost.db?cache=shared")
+ sqldb, err := db.SqlDB(dbPath)
+ if err != nil {
+ return fmt.Errorf("opening boost sqlite db: %w", err)
+ }
+
+ mdbPath := path.Join(repoDir, "migrate-curio.db?cache=shared")
+ mdb, err := db.SqlDB(mdbPath)
+ if err != nil {
+ return fmt.Errorf("opening migrate sqlite db: %w", err)
+ }
+
+ _, err = mdb.Exec(`CREATE TABLE IF NOT EXISTS Deals (
+ ID TEXT UNIQUE,
+ DB BOOL NOT NULL DEFAULT FALSE,
+ LID BOOL NOT NULL DEFAULT FALSE,
+ Pipeline BOOL NOT NULL DEFAULT FALSE
+ );`)
+ if err != nil {
+ return fmt.Errorf("failed to create migration table: %w", err)
+ }
+
+ mActor, err := full.StateGetActor(ctx, maddr, ltypes.EmptyTSK)
+ if err != nil {
+ return fmt.Errorf("getting actor for the miner %s: %w", maddr, err)
+ }
+ astore := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(full)))
+ mas, err := miner.Load(astore, mActor)
+ if err != nil {
+ return fmt.Errorf("loading miner actor state %s: %w", maddr, err)
+ }
+ liveSectors, err := miner.AllPartSectors(mas, miner.Partition.LiveSectors)
+ if err != nil {
+ return fmt.Errorf("getting live sector sets for miner %s: %w", maddr, err)
+ }
+ unProvenSectors, err := miner.AllPartSectors(mas, miner.Partition.UnprovenSectors)
+ if err != nil {
+ return fmt.Errorf("getting unproven sector sets for miner %s: %w", maddr, err)
+ }
+ activeSectors, err := bitfield.MergeBitFields(liveSectors, unProvenSectors)
+ if err != nil {
+ return fmt.Errorf("merging bitfields to generate all sealed sectors on miner %s: %w", maddr, err)
+ }
+
+ // Migrate Boost deals
+ if err := migrateBoostDeals(ctx, activeSectors, maddr, hdb, sqldb, mdb); err != nil {
+ return xerrors.Errorf("failed to migrate boost deals: %w", err)
+ }
+
+ // Migrate Legacy deal
+ if err := migrateLegacyDeals(ctx, full, activeSectors, maddr, hdb, mds, mdb); err != nil {
+ return xerrors.Errorf("failed to migrate legacy deals: %w", err)
+ }
+
+ // Migrate Direct deals
+ if err := migrateDDODeals(ctx, full, activeSectors, maddr, hdb, sqldb, mdb); err != nil {
+ return xerrors.Errorf("failed to migrate DDO deals: %w", err)
+ }
+
+ return nil
+}
+
+func migrateBoostDeals(ctx context.Context, activeSectors bitfield.BitField, maddr address.Address, hdb *harmonydb.DB, sqldb, mdb *sql.DB) error {
+ sdb := db.NewDealsDB(sqldb)
+
+ mid, err := address.IDFromAddress(maddr)
+ if err != nil {
+ return fmt.Errorf("address.IDFromAddress: %s", err)
+ }
+
+ aDeals, err := sdb.ListActive(ctx)
+ if err != nil {
+ return err
+ }
+
+ cDeals, err := sdb.ListCompleted(ctx)
+ if err != nil {
+ return err
+ }
+
+ deals := append(aDeals, cDeals...)
+
+ for i, deal := range deals {
+ if i > 0 && i%100 == 0 {
+ fmt.Printf("Migrating Boost Deals: %d / %d (%0.2f%%)\n", i, len(deals), float64(i)/float64(len(deals))*100)
+ }
+
+ llog := log.With("Boost Deal", deal.DealUuid.String())
+ // Skip deals which are before add piece
+ if deal.Checkpoint < dealcheckpoints.AddedPiece {
+ llog.Infow("Skipping as checkpoint is below add piece")
+ continue
+ }
+
+ // Skip deals which do not have chain deal ID
+ if deal.ChainDealID == 0 {
+ llog.Infow("Skipping as chain deal ID is 0")
+ continue
+ }
+
+ // Skip deals which do not have retryable error
+ if deal.Retry == types.DealRetryFatal {
+ llog.Infow("Skipping as deal retry is fatal")
+ continue
+ }
+
+ // SKip sector 0. This might cause some deals to not migrate but
+ // that is better than migrating faulty deals
+ if deal.SectorID == 0 {
+ llog.Infow("Skipping as sector ID is 0")
+ continue
+ }
+
+ // Skip if the sector for the deal is not alive
+ ok, err := activeSectors.IsSet(uint64(deal.SectorID))
+ if err != nil {
+ return err
+ }
+ if !ok {
+ llog.Infof("Skipping as sector %d is not alive anymore", deal.SectorID)
+ continue
+ }
+
+ // Skip if already migrated
+ var a, b, c bool
+ err = mdb.QueryRow(`SELECT DB, LID, Pipeline FROM Deals WHERE ID = ?`, deal.DealUuid.String()).Scan(&a, &b, &c)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("deal: %s: failed to check migration status: %w", deal.DealUuid.String(), err)
+ }
+ }
+ if a && b && c {
+ llog.Infow("Skipped as this deal is already migrated")
+ continue
+ }
+
+ propJson, err := json.Marshal(deal.ClientDealProposal.Proposal)
+ if err != nil {
+ return fmt.Errorf("deal: %s: json.Marshal(piece.DealProposal): %s", deal.DealUuid.String(), err)
+ }
+
+ sigByte, err := deal.ClientDealProposal.ClientSignature.MarshalBinary()
+ if err != nil {
+ return fmt.Errorf("deal: %s: marshal client signature: %s", deal.DealUuid.String(), err)
+ }
+
+ prop := deal.ClientDealProposal.Proposal
+
+ sProp, err := deal.SignedProposalCid()
+ if err != nil {
+ return err
+ }
+
+ // de-serialize transport opaque token
+ tInfo := &transportTypes.HttpRequest{}
+ if err := json.Unmarshal(deal.Transfer.Params, tInfo); err != nil {
+ return fmt.Errorf("deal: %s: failed to de-serialize transport params bytes '%s': %s", deal.DealUuid.String(), string(deal.Transfer.Params), err)
+ }
+
+ hdr := http.Header{}
+ for k, v := range tInfo.Headers {
+ hdr.Add(k, v)
+ }
+
+ headers, err := json.Marshal(hdr)
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to marshal headers: %s", deal.DealUuid.String(), err)
+ }
+
+ // Cbor marshal the Deal Label manually as non-string label will result in "" with JSON marshal
+ label := prop.Label
+ buf := new(bytes.Buffer)
+ err = label.MarshalCBOR(buf)
+ if err != nil {
+ return fmt.Errorf("cbor marshal label: %w", err)
+
+ }
+
+ _, err = hdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) {
+ // Add deal to HarmonyDB
+ if !a {
+ _, err = tx.Exec(`INSERT INTO market_mk12_deals (uuid, sp_id, signed_proposal_cid,
+ proposal_signature, proposal, piece_cid,
+ piece_size, offline, verified, start_epoch, end_epoch,
+ client_peer_id, fast_retrieval, announce_to_ipni, url, url_headers, chain_deal_id, publish_cid, created_at, label)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
+ ON CONFLICT (uuid) DO NOTHING`,
+ deal.DealUuid.String(), mid, sProp.String(), sigByte, propJson, prop.PieceCID.String(),
+ prop.PieceSize, deal.IsOffline, prop.VerifiedDeal, prop.StartEpoch, prop.EndEpoch, deal.ClientPeerID.String(),
+ deal.FastRetrieval, deal.AnnounceToIPNI, tInfo.URL, headers, int64(deal.ChainDealID), deal.PublishCID.String(), deal.CreatedAt, buf.Bytes())
+
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add the deal to harmonyDB: %w", deal.DealUuid.String(), err)
+ }
+
+ // Mark deal added to harmonyDB
+ _, err = mdb.Exec(`INSERT INTO Deals (ID, DB, LID) VALUES (?, TRUE, FALSE) ON CONFLICT(ID) DO NOTHING`, deal.DealUuid.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark deal migrated: %w", deal.DealUuid.String(), err)
+ }
+ }
+
+ if !b {
+ // Add LID details to pieceDeal in HarmonyDB
+ _, err = tx.Exec(`SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
+ deal.DealUuid.String(), prop.PieceCID.String(), true, mid, deal.SectorID, deal.Offset,
+ prop.PieceSize, deal.NBytesReceived, false, false, deal.ChainDealID)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to update piece metadata and piece deal: %w", deal.DealUuid.String(), err)
+ }
+
+ // Mark deal added to pieceDeal in HarmonyDB
+ _, err = mdb.Exec(`UPDATE Deals SET LID = TRUE WHERE ID = ?`, deal.DealUuid.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark deal LID migrated: %w", deal.DealUuid.String(), err)
+ }
+ }
+
+ if !c {
+ // Check if we can index and announce i.e. we have unsealed copy
+ var exists bool
+ err = tx.QueryRow(`SELECT EXISTS (SELECT 1 FROM sector_location WHERE miner_id = $1
+ AND sector_num = $2
+ AND sector_filetype = 1);`, mid, deal.SectorID).Scan(&exists)
+ if err != nil {
+ return false, fmt.Errorf("seal: %s: failed to check if sector is unsealed: %w", deal.DealUuid.String(), err)
+ }
+
+ if exists {
+ var proof abi.RegisteredSealProof
+ err = tx.QueryRow(`SELECT reg_seal_proof FROM sectors_meta WHERE sp_id = $1 AND sector_num = $2`, mid, deal.SectorID).Scan(&proof)
+ if err != nil {
+ llog.Errorw("failed to get sector proof", "error", err, "deal", deal.DealUuid.String(), "sector", deal.SectorID, "miner", mid)
+ return false, nil
+ }
+
+ // Add deal to mk12 pipeline in Curio for indexing and announcement
+ _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline_migration (
+ uuid, sp_id, piece_cid, piece_size, raw_size, sector, reg_seal_proof, sector_offset, should_announce
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (uuid) DO NOTHING`,
+ deal.DealUuid.String(), mid, prop.PieceCID.String(), prop.PieceSize, deal.NBytesReceived,
+ deal.SectorID, proof, deal.Offset, deal.AnnounceToIPNI)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add deal to pipeline for indexing and announcing: %w", deal.DealUuid.String(), err)
+ }
+ } else {
+ llog.Infof("Skipping indexing as sector %d is not unsealed", deal.SectorID)
+ }
+ }
+ return true, nil
+ }, harmonydb.OptionRetry())
+ if err != nil {
+ return err
+ }
+
+ }
+
+ return nil
+}
+
+func migrateLegacyDeals(ctx context.Context, full v1api.FullNode, activeSectors bitfield.BitField, maddr address.Address, hdb *harmonydb.DB, ds datastore.Batching, mdb *sql.DB) error {
+ mid, err := address.IDFromAddress(maddr)
+ if err != nil {
+ return fmt.Errorf("address.IDFromAddress: %s", err)
+ }
+
+ // Get the deals FSM
+ provDS := namespace.Wrap(ds, datastore.NewKey("/deals/provider"))
+ deals, migrate, err := vfsm.NewVersionedFSM(provDS, fsm.Parameters{
+ StateType: legacytypes.MinerDeal{},
+ StateKeyField: "State",
+ }, nil, "2")
+ if err != nil {
+ return fmt.Errorf("reading legacy deals from datastore: %w", err)
+ }
+
+ err = migrate(ctx)
+ if err != nil {
+ return fmt.Errorf("running provider fsm migration script: %w", err)
+ }
+
+ lm := legacy.NewLegacyDealsManager(deals)
+ go lm.Run(ctx)
+ // Wait for 5 seconds
+ time.Sleep(time.Second * 5)
+
+ legacyDeals, err := lm.ListDeals()
+ if err != nil {
+ return fmt.Errorf("getting legacy deals: %w", err)
+ }
+
+ head, err := full.ChainHead(ctx)
+ if err != nil {
+ return err
+ }
+
+ for i, deal := range legacyDeals {
+ if i > 0 && i%100 == 0 {
+ fmt.Printf("Migrating Legacy Deals: %d / %d (%0.2f%%)\n", i, len(legacyDeals), float64(i)/float64(len(legacyDeals))*100)
+ }
+ llog := log.With("Boost Deal", deal.ProposalCid.String())
+ // Skip deals which do not have chain deal ID
+ if deal.DealID == 0 {
+ llog.Infow("Skipping as chain deal ID is 0")
+ continue
+ }
+
+ // SKip sector 0. This might cause some deals to not migrate but
+ // that is better than migrating faulty deals
+ if deal.SectorNumber == 0 {
+ llog.Infow("Skipping as sector ID is 0")
+ continue
+ }
+
+ // Skip expired legacy deals
+ if deal.ClientDealProposal.Proposal.EndEpoch < head.Height() {
+ llog.Infow("Deal end epoch is lower than current height")
+ continue
+ }
+
+ // Skip if the sector for the deal is not alive
+ ok, err := activeSectors.IsSet(uint64(deal.SectorNumber))
+ if err != nil {
+ return err
+ }
+ if !ok {
+ llog.Infof("Skipping as sector %d is not alive anymore", deal.SectorNumber)
+ continue
+ }
+
+ // Skip if already migrated
+ var a, b, c bool
+ err = mdb.QueryRow(`SELECT DB, LID, Pipeline FROM Deals WHERE ID = ?`, deal.ProposalCid.String()).Scan(&a, &b, &c)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("deal: %s: failed to check migration status: %w", deal.ProposalCid.String(), err)
+ }
+ }
+ if a && b && c {
+ llog.Infow("Skipped as this deal is already migrated")
+ continue
+ }
+
+ propJson, err := json.Marshal(deal.ClientDealProposal.Proposal)
+ if err != nil {
+ return fmt.Errorf("deal: %s: json.Marshal(piece.DealProposal): %s", deal.ProposalCid.String(), err)
+ }
+
+ sigByte, err := deal.ClientDealProposal.ClientSignature.MarshalBinary()
+ if err != nil {
+ return fmt.Errorf("deal: %s: marshal client signature: %s", deal.ProposalCid.String(), err)
+ }
+
+ prop := deal.ClientDealProposal.Proposal
+
+ _, err = hdb.Exec(ctx, `INSERT INTO market_legacy_deals (signed_proposal_cid, sp_id, client_peer_id,
+ proposal_signature, proposal, piece_cid,
+ piece_size, verified, start_epoch, end_epoch,
+ publish_cid, chain_deal_id, fast_retrieval, created_at, sector_num)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
+ ON CONFLICT (sp_id, piece_cid, signed_proposal_cid) DO NOTHING`,
+ deal.ProposalCid, mid, deal.Client.String(), sigByte, propJson, prop.PieceCID.String(),
+ prop.PieceSize, prop.VerifiedDeal, prop.StartEpoch, prop.EndEpoch, deal.PublishCid.String(),
+ deal.DealID, deal.FastRetrieval, deal.CreationTime.Time(), deal.SectorNumber)
+
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to add the legacy deal to harmonyDB: %w", deal.ProposalCid.String(), err)
+ }
+
+ // Mark deal added to harmonyDB
+ _, err = mdb.Exec(`INSERT INTO Deals (ID, DB, LID) VALUES (?, TRUE, TRUE) ON CONFLICT(ID) DO NOTHING`, deal.ProposalCid.String())
+ if err != nil {
+ return fmt.Errorf("deal: %s: failed to mark deal migrated: %w", deal.ProposalCid.String(), err)
+ }
+ }
+
+ return nil
+}
+
+func migrateDDODeals(ctx context.Context, full v1api.FullNode, activeSectors bitfield.BitField, maddr address.Address, hdb *harmonydb.DB, sqldb, mdb *sql.DB) error {
+ ddb := db.NewDirectDealsDB(sqldb)
+
+ mid, err := address.IDFromAddress(maddr)
+ if err != nil {
+ return fmt.Errorf("address.IDFromAddress: %s", err)
+ }
+
+ deals, err := ddb.ListAll(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to get all DDO deals: %w", err)
+ }
+
+ for i, deal := range deals {
+ if i > 0 && i%100 == 0 {
+ fmt.Printf("Migrating DDO Deals: %d / %d (%0.2f%%)\n", i, len(deals), float64(i)/float64(len(deals))*100)
+ }
+ llog := log.With("DDO Deal", deal.ID.String())
+ if deal.Err != "" && deal.Retry == types.DealRetryFatal {
+ llog.Infow("Skipping as deal retry is fatal")
+ continue
+ }
+
+ if deal.Checkpoint < dealcheckpoints.AddedPiece {
+ llog.Infow("Skipping as checkpoint is below add piece")
+ continue
+ }
+
+ claim, err := full.StateGetClaim(ctx, maddr, verifreg9types.ClaimId(deal.AllocationID), ltypes.EmptyTSK)
+ if err != nil {
+ return fmt.Errorf("deal: %s: error getting the claim status: %w", deal.ID.String(), err)
+ }
+ if claim == nil {
+ llog.Infow("Skipping as checkpoint is below add piece")
+ continue
+ }
+ if claim.Sector != deal.SectorID {
+ return fmt.Errorf("deal: %s: sector mismatch for deal", deal.ID.String())
+ }
+
+ // Skip if the sector for the deal is not alive
+ ok, err := activeSectors.IsSet(uint64(deal.SectorID))
+ if err != nil {
+ return err
+ }
+ if !ok {
+ llog.Infow("Skipping as sector ID is 0")
+ continue
+ }
+
+ // Skip if already migrated
+ var a, b, c bool
+ err = mdb.QueryRow(`SELECT DB, LID, Pipeline FROM Deals WHERE ID = ?`, deal.ID.String()).Scan(&a, &b, &c)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("deal: %s: failed to check migration status: %w", deal.ID.String(), err)
+ }
+ }
+ if a && b && c {
+ continue
+ }
+
+ _, err = hdb.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) {
+ if !a {
+ // Add DDO deal to harmonyDB
+ _, err = tx.Exec(`INSERT INTO market_direct_deals (uuid, sp_id, created_at, client, offline, verified,
+ start_epoch, end_epoch, allocation_id, piece_cid, piece_size, fast_retrieval, announce_to_ipni)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+ ON CONFLICT (uuid) DO NOTHING`,
+ deal.ID.String(), mid, deal.CreatedAt, deal.Client.String(), true, true, deal.StartEpoch, deal.EndEpoch, deal.AllocationID,
+ deal.PieceCID.String(), deal.PieceSize, true, true)
+
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add the DDO deal to harmonyDB: %w", deal.ID.String(), err)
+ }
+
+ // Mark deal added to harmonyDB
+ _, err = mdb.Exec(`INSERT INTO Deals (ID, DB, LID) VALUES (?, TRUE, FALSE) ON CONFLICT(ID) DO NOTHING`, deal.ID.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark DDO deal migrated: %w", deal.ID.String(), err)
+ }
+ }
+
+ if !b {
+ // Add LID details to pieceDeal in HarmonyDB
+ _, err = tx.Exec(`SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
+ deal.ID.String(), deal.PieceCID.String(), false, mid, deal.SectorID, deal.Offset, deal.PieceSize, deal.InboundFileSize, false)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to update piece metadata and piece deal for DDO deal %s: %w", deal.ID.String(), deal.ID.String(), err)
+ }
+
+ // Mark deal added to pieceDeal in HarmonyDB
+ _, err = mdb.Exec(`UPDATE Deals SET LID = TRUE WHERE ID = ?`, deal.ID.String())
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to mark deal LID migrated: %w", deal.ID.String(), err)
+ }
+ }
+
+ // TODO: Confirm if using the mk12 pipeline will have any impact for DDO deals
+ if !c {
+ // Check if we can index and announce i.e. we have unsealed copy
+ var exists bool
+ err = tx.QueryRow(`SELECT EXISTS (SELECT 1 FROM sector_location WHERE miner_id = $1
+ AND sector_num = $2
+ AND sector_filetype = 1);`, mid, deal.SectorID).Scan(&exists)
+ if err != nil {
+ return false, fmt.Errorf("seal: %s: failed to check if sector is unsealed: %w", deal.ID.String(), err)
+ }
+
+ if exists {
+ var proof abi.RegisteredSealProof
+ err = tx.QueryRow(`SELECT reg_seal_proof FROM sectors_meta WHERE sp_id = $1 AND sector_num = $2`, mid, deal.SectorID).Scan(&proof)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to get sector proof: %w", deal.ID.String(), err)
+ }
+
+ // Add deal to mk12 pipeline in Curio for indexing and announcement
+ _, err = tx.Exec(`INSERT INTO market_mk12_deal_pipeline_migration (
+ uuid, sp_id, piece_cid, piece_size, raw_size, sector, reg_seal_proof, sector_offset, should_announce
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (uuid) DO NOTHING`,
+ deal.ID.String(), mid, deal.PieceCID.String(), deal.PieceSize, deal.InboundFileSize,
+ deal.SectorID, proof, deal.Offset, true)
+ if err != nil {
+ return false, fmt.Errorf("deal: %s: failed to add DDO deal to pipeline for indexing and announcing: %w", deal.ID.String(), err)
+ }
+ } else {
+ llog.Infof("Skipping indexing as sector %d is not unsealed", deal.SectorID)
+ }
+ }
+ return true, nil
+ }, harmonydb.OptionRetry())
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/cmd/migrate-curio/yugabyte.go b/cmd/migrate-curio/yugabyte.go
new file mode 100644
index 000000000..0624d4df8
--- /dev/null
+++ b/cmd/migrate-curio/yugabyte.go
@@ -0,0 +1,139 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/filecoin-project/boost/extern/boostd-data/yugabyte"
+ "github.com/urfave/cli/v2"
+ "github.com/yugabyte/pgx/v4/pgxpool"
+)
+
+var cleanupLIDCmd = &cli.Command{
+ Name: "cleanup",
+ Description: "Removes the indexes and other metadata from LID",
+ Usage: "migrate-curio cleanup",
+ Subcommands: []*cli.Command{
+ cleanupLevelDBCmd,
+ cleanupYugabyteDBCmd,
+ },
+}
+
+var cleanupYugabyteDBCmd = &cli.Command{
+ Name: "yugabyte",
+ Description: "Removes the indexes and other metadata from Yugabyte based LID store",
+ Usage: "migrate-curio cleanup yugabyte",
+ Before: before,
+ Flags: []cli.Flag{
+ &cli.StringSliceFlag{
+ Name: "hosts",
+ Usage: "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'",
+ Required: true,
+ },
+ &cli.StringFlag{
+ Name: "username",
+ Usage: "yugabyte username to connect to over cassandra interface eg 'cassandra'",
+ },
+ &cli.StringFlag{
+ Name: "password",
+ Usage: "yugabyte password to connect to over cassandra interface eg 'cassandra'",
+ },
+ &cli.StringFlag{
+ Name: "connect-string",
+ Usage: "postgres connect string eg 'postgresql://postgres:postgres@localhost'",
+ Required: true,
+ },
+ &cli.BoolFlag{
+ Name: "i-know-what-i-am-doing",
+ Usage: "confirmation flag",
+ Value: false,
+ },
+ },
+ Action: func(cctx *cli.Context) error {
+ if !cctx.Bool("i-know-what-i-am-doing") {
+ return fmt.Errorf("please use --i-know-what-i-am-doing flag to confirm. THIS CANNOT BE UNDONE.")
+ }
+
+ // Create a yugabyte data service
+ settings := yugabyte.DBSettings{
+ Hosts: cctx.StringSlice("hosts"),
+ Username: cctx.String("username"),
+ Password: cctx.String("password"),
+ ConnectString: cctx.String("connect-string"),
+ }
+
+ settings.CQLTimeout = 60
+
+ cluster := yugabyte.NewCluster(settings)
+
+ session, err := cluster.CreateSession()
+ if err != nil {
+ return fmt.Errorf("creating yugabyte cluster: %w", err)
+ }
+
+ keyspace := "idx"
+
+ // Drop all indexes
+ var indexName string
+ indexesQuery := fmt.Sprintf("SELECT index_name FROM system_schema.indexes WHERE keyspace_name='%s';", keyspace)
+ iter := session.Query(indexesQuery).Iter()
+
+ for iter.Scan(&indexName) {
+ dropIndexQuery := fmt.Sprintf("DROP INDEX %s.%s;", keyspace, indexName)
+ fmt.Println("Executing:", dropIndexQuery)
+ if err := session.Query(dropIndexQuery).Exec(); err != nil {
+ return fmt.Errorf("failed to drop index %s: %w", indexName, err)
+ }
+ }
+ if err := iter.Close(); err != nil {
+ return fmt.Errorf("failed to iterate over indexes: %w", err)
+ }
+
+ // Query to get all tables in the 'idx' keyspace
+ tableQuesry := fmt.Sprintf(`SELECT table_name FROM system_schema.tables WHERE keyspace_name='%s';`, keyspace)
+ iter = session.Query(tableQuesry).Iter()
+
+ var tableName string
+ for iter.Scan(&tableName) {
+ dropQuery := fmt.Sprintf("DROP TABLE idx.%s", tableName)
+ fmt.Println("Executing:", dropQuery)
+
+ err := session.Query(dropQuery).Exec()
+ if err != nil {
+ return fmt.Errorf("failed to drop table %s: %w", tableName, err)
+ }
+ }
+
+ if err := iter.Close(); err != nil {
+ return fmt.Errorf("error closing iterator: %v", err)
+ }
+
+ fmt.Println("All tables in keyspace 'idx' have been dropped.")
+
+ // Drop the keyspace
+ dropKeyspaceQuery := fmt.Sprintf("DROP KEYSPACE %s;", keyspace)
+ fmt.Println("Executing:", dropKeyspaceQuery)
+ if err := session.Query(dropKeyspaceQuery).Exec(); err != nil {
+ return fmt.Errorf("failed to drop keyspace: %w", err)
+ }
+
+ fmt.Println("Keyspace dropped successfully.")
+
+ // Create connection pool to postgres interface
+ db, err := pgxpool.Connect(cctx.Context, settings.ConnectString)
+ if err != nil {
+ return err
+ }
+
+ _, err = db.Exec(cctx.Context, `DROP TABLE IF EXISTS PieceTracker CASCADE;`)
+ if err != nil {
+ return err
+ }
+
+ _, err = db.Exec(cctx.Context, `DROP TABLE IF EXISTS PieceFlagged CASCADE;`)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ },
+}
diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
index 859bfc871..5dfd09678 100644
--- a/documentation/en/api-v1-methods.md
+++ b/documentation/en/api-v1-methods.md
@@ -20,6 +20,7 @@
* [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
* [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal)
* [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
+ * [BoostIndexerRemoveAll](#boostindexerremoveall)
* [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid)
* [BoostOfflineDealWithData](#boostofflinedealwithdata)
* [I](#i)
@@ -374,7 +375,7 @@ Response:
```
### BoostIndexerAnnounceAllDeals
-There are not yet any comments for this method.
+
Perms: admin
@@ -513,6 +514,20 @@ Response:
]
```
+### BoostIndexerRemoveAll
+There are not yet any comments for this method.
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+[
+ null
+]
+```
+
### BoostLegacyDealByProposalCid
diff --git a/go.mod b/go.mod
index 4b2f79b40..c34934c48 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/filecoin-project/boost
-go 1.22
+go 1.22.3
replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
@@ -21,6 +21,7 @@ require (
github.com/fatih/color v1.16.0
github.com/filecoin-project/boost-graphsync v0.13.12
github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f
+ github.com/filecoin-project/curio v1.23.2-0.20241017101155-587dd6361137
github.com/filecoin-project/dagstore v0.7.0
github.com/filecoin-project/go-address v1.2.0
github.com/filecoin-project/go-bitfield v0.2.4
@@ -30,13 +31,13 @@ require (
github.com/filecoin-project/go-ds-versioning v0.1.2
github.com/filecoin-project/go-fil-commcid v0.2.0
github.com/filecoin-project/go-fil-commp-hashhash v0.2.0
- github.com/filecoin-project/go-jsonrpc v0.6.0
+ github.com/filecoin-project/go-jsonrpc v0.6.1-0.20240820160949-2cfe810e5d2f
github.com/filecoin-project/go-padreader v0.0.1
github.com/filecoin-project/go-paramfetch v0.0.4
github.com/filecoin-project/go-state-types v0.15.0-rc1
github.com/filecoin-project/go-statemachine v1.0.3
github.com/filecoin-project/go-statestore v0.2.0
- github.com/filecoin-project/lotus v1.30.0-rc2
+ github.com/filecoin-project/lotus v1.30.0-rc2.0.20241016173451-c07d2f73e436
github.com/filecoin-project/specs-actors v0.9.15
github.com/filecoin-project/specs-actors/v2 v2.3.6
github.com/gbrlsnchs/jwt/v3 v3.0.1
@@ -107,6 +108,7 @@ require (
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11
github.com/whyrusleeping/cbor-gen v0.1.2
+ github.com/yugabyte/pgx/v4 v4.14.5
go.opencensus.io v0.24.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/trace v1.28.0
@@ -174,6 +176,7 @@ require (
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect
github.com/filecoin-project/go-amt-ipld/v4 v4.4.0 // indirect
github.com/filecoin-project/go-clock v0.1.0 // indirect
+ github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 // indirect
github.com/filecoin-project/go-commp-utils/v2 v2.1.0 // indirect
github.com/filecoin-project/go-crypto v0.1.0 // indirect
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8 // indirect
@@ -266,7 +269,7 @@ require (
github.com/jessevdk/go-flags v1.4.0 // indirect
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
github.com/josharian/intern v1.0.0 // indirect
- github.com/kilic/bls12-381 v0.1.0 // indirect
+ github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
@@ -347,7 +350,7 @@ require (
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
- github.com/triplewz/poseidon v0.0.1 // indirect
+ github.com/triplewz/poseidon v0.0.2-0.20240407130934-5265fab9d889 // indirect
github.com/twmb/murmur3 v1.1.6 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
@@ -359,7 +362,6 @@ require (
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
github.com/yugabyte/gocql v0.0.0-20230831121436-1e2272bb6bb6 // indirect
- github.com/yugabyte/pgx/v4 v4.14.5 // indirect
github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // indirect
github.com/zondax/hid v0.9.2 // indirect
github.com/zondax/ledger-filecoin-go v0.11.1 // indirect
@@ -390,5 +392,3 @@ require (
lukechampine.com/blake3 v1.3.0 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
-
-replace github.com/KarpelesLab/reflink => github.com/magik6k/reflink v1.0.2-patch1
diff --git a/go.sum b/go.sum
index 87ea76bd7..6bf37f9e8 100644
--- a/go.sum
+++ b/go.sum
@@ -332,6 +332,8 @@ github.com/filecoin-project/boost-graphsync v0.13.12 h1:fAGaHRIYoN6cPMs2ChVymio8
github.com/filecoin-project/boost-graphsync v0.13.12/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk=
github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f h1:8dd0yAadyeOL5Qd42XhEwD60UKvIFkY2MLhef/IaeOk=
github.com/filecoin-project/boost/extern/boostd-data v0.0.0-20231124125934-3233c510357f/go.mod h1:MyzvfYWAH0OAyf95TLUWYq3cO3vm/TVzDS57GKQi47o=
+github.com/filecoin-project/curio v1.23.2-0.20241017101155-587dd6361137 h1:vb2R0gbX8yN/Qod0CKL+u2xtu/KuR817AhObMnDYkXY=
+github.com/filecoin-project/curio v1.23.2-0.20241017101155-587dd6361137/go.mod h1:mQ4yfT238Yu0bsRBfxsqnQb9uRshkvUM5wIllu0QzV8=
github.com/filecoin-project/dagstore v0.7.0 h1:IS0R+69za8dguYWeqz/MI+nb7ONpk03tAkxPCBXEKm0=
github.com/filecoin-project/dagstore v0.7.0/go.mod h1:YKn4qXih+/2xQWpfJsaKGOi4POw5vH5grDmfPCCnx8g=
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
@@ -357,6 +359,8 @@ github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9g
github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs=
github.com/filecoin-project/go-commp-utils v0.1.4 h1:/WSsrAb0xupo+aRWRyD80lRUXAXJvYoTgDQS1pYZ1Mk=
github.com/filecoin-project/go-commp-utils v0.1.4/go.mod h1:Sekocu5q9b4ECAUFu853GFUbm8I7upAluummHFe2kFo=
+github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 h1:jAG2g1Fs/qoDSSaI8JaP/KmqR+QQ8IVQ6k9xKONa72M=
+github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8/go.mod h1:kU2KuSPLB+Xz4FEbVE0abzSN4l6irZ8tqgcYWPVDftU=
github.com/filecoin-project/go-commp-utils/v2 v2.1.0 h1:KWNRalUp2bhN1SW7STsJS2AHs9mnfGKk9LnQgzDe+gI=
github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj/gysLNu26kYII83IV5iNrAO9iI=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
@@ -382,8 +386,8 @@ github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGy
github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g=
github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 h1:nYs6OPUF8KbZ3E8o9p9HJnQaE8iugjHR5WYVMcicDJc=
github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0/go.mod h1:s0qiHRhFyrgW0SvdQMSJFQxNa4xEIG5XvqCBZUEgcbc=
-github.com/filecoin-project/go-jsonrpc v0.6.0 h1:/fFJIAN/k6EgY90m7qbyfY28woMwyseZmh2gVs5sYjY=
-github.com/filecoin-project/go-jsonrpc v0.6.0/go.mod h1:/n/niXcS4ZQua6i37LcVbY1TmlJR0UIK9mDFQq2ICek=
+github.com/filecoin-project/go-jsonrpc v0.6.1-0.20240820160949-2cfe810e5d2f h1:0FMH/uwBH7RinWrE+TkiOotYoqxSM54teKx/olJ/cWs=
+github.com/filecoin-project/go-jsonrpc v0.6.1-0.20240820160949-2cfe810e5d2f/go.mod h1:/n/niXcS4ZQua6i37LcVbY1TmlJR0UIK9mDFQq2ICek=
github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs=
github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
github.com/filecoin-project/go-paramfetch v0.0.4 h1:H+Me8EL8T5+79z/KHYQQcT8NVOzYVqXIi7nhb48tdm8=
@@ -405,8 +409,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd
github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo=
github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus=
github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8=
-github.com/filecoin-project/lotus v1.30.0-rc2 h1:LLzMnb6dqxN5QHj4IAvDpFPYp8InXY8fvcTGr4uhpnw=
-github.com/filecoin-project/lotus v1.30.0-rc2/go.mod h1:gXQFTK6OpJIjg2yWnYsf0awszREDffb/X+LPCDmZkwI=
+github.com/filecoin-project/lotus v1.30.0-rc2.0.20241016173451-c07d2f73e436 h1:hr/BnP02MP5I00CTvhed3eJILDksobzhmbi9zwwvylg=
+github.com/filecoin-project/lotus v1.30.0-rc2.0.20241016173451-c07d2f73e436/go.mod h1:SzG9YCzgmShS36hGG8PUyASa6pf91zlMsiIARxuCXQQ=
github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM=
github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg=
github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
@@ -1075,8 +1079,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
-github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4=
-github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig=
+github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 h1:xWK4TZ4bRL05WQUU/3x6TG1l+IYAqdXpAeSLt/zZJc4=
+github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4/go.mod h1:tlkavyke+Ac7h8R3gZIjI5LKBcvMlSWnXNMgT3vZXo8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -1742,8 +1746,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -1817,6 +1821,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
+github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb h1:78YgPq3NbWnO4xyNhLsn2zitc7NiZpjQZ560rsxVLm4=
+github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb/go.mod h1:YnCEHk4UlWROjM3YzC2pbSTq+iynM3ZaLoVDUI8QGpE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
@@ -1881,8 +1887,8 @@ github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0h
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/triplewz/poseidon v0.0.1 h1:G5bdkTzb9R5K5Dd3DIzBCp7rAErP1zWH0LW7Ip6bxIA=
-github.com/triplewz/poseidon v0.0.1/go.mod h1:QYG1d0B4YZD7TgF6qZndTTu4rxUGFCCZAQRDanDj+9c=
+github.com/triplewz/poseidon v0.0.2-0.20240407130934-5265fab9d889 h1:cbYPZOEknyV/Gyud82ebTPiciOnVSv6tiMCQi5Y+mAs=
+github.com/triplewz/poseidon v0.0.2-0.20240407130934-5265fab9d889/go.mod h1:fmoxtMcbtMUjlSJmpuS3Wk/oKSvdJpIp9YWRbsOu3T0=
github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
@@ -2328,7 +2334,6 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -2344,7 +2349,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
diff --git a/go.work b/go.work
index abdbb01e4..013a26e98 100644
--- a/go.work
+++ b/go.work
@@ -1,4 +1,4 @@
-go 1.22
+go 1.22.3
use (
.
diff --git a/go.work.sum b/go.work.sum
index a0111e92f..d36365de6 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -1095,6 +1095,8 @@ github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKz
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7 h1:mreN1m/5VJ/Zc3b4pzj9qU6D9SRQ6Vm+3KfI328t3S8=
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM=
+github.com/KarpelesLab/reflink v1.0.1 h1:d+tdjliwOCqvub9bl0Y02GxahWkNqejNb3TZTTUcQWA=
+github.com/KarpelesLab/reflink v1.0.1/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk=
github.com/Masterminds/glide v0.13.2 h1:M5MOH04TyRiMBVeWHbifqTpnauxWINIubTCOkhXh+2g=
@@ -2312,6 +2314,8 @@ github.com/lyft/protoc-gen-validate v0.0.0-20180911180927-64fcb82c878e/go.mod h1
github.com/lyft/protoc-gen-validate v0.0.13 h1:KNt/RhmQTOLr7Aj8PsJ7mTronaFyx80mRTT9qF261dA=
github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/magik6k/reflink v1.0.2-patch1 h1:NXSgQugcESI8Z/jBtuAI83YsZuRauY9i9WOyOnJ7Vns=
+github.com/magik6k/reflink v1.0.2-patch1/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok=
github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18=
github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE=
@@ -2663,6 +2667,7 @@ github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52 h1:RnWNS9Hlm8BI
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
@@ -3257,6 +3262,7 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go
index 0976ad1a2..0b564bee4 100644
--- a/indexprovider/wrapper.go
+++ b/indexprovider/wrapper.go
@@ -18,11 +18,11 @@ import (
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
chainTypes "github.com/filecoin-project/lotus/chain/types"
"github.com/google/uuid"
- cbor "github.com/ipfs/go-ipld-cbor"
- "go.uber.org/fx"
-
"github.com/ipfs/go-datastore"
+ cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipld/go-ipld-prime"
+ "github.com/ipni/go-libipni/ingest/schema"
+ "go.uber.org/fx"
"github.com/filecoin-project/boost/db"
bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types"
@@ -867,3 +867,52 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u
}
return annCid, err
}
+
+func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+ var allAds []*schema.Advertisement
+ _, ad, err := w.prov.GetLatestAdv(ctx)
+ if err != nil {
+ return nil, err
+ }
+ allAds = append(allAds, ad)
+
+ prev, err := cid.Parse(ad.PreviousID.String())
+ if err != nil {
+ return nil, err
+ }
+
+ for prev != cid.Undef {
+ ad, err := w.prov.GetAdv(ctx, prev)
+ if err != nil {
+ return nil, err
+ }
+
+ prev, err = cid.Parse(ad.PreviousID.String())
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ var entryAds []*schema.Advertisement
+
+ for _, ad := range allAds {
+ if !ad.IsRm {
+ entryAds = append(entryAds, ad)
+ }
+ }
+
+ var newAds []cid.Cid
+
+ for _, ad := range entryAds {
+ a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID)
+ if err != nil {
+ if !errors.Is(err, provider.ErrContextIDNotFound) {
+ return nil, fmt.Errorf("failed to publish the removal ad: %w", err)
+ }
+ }
+ newAds = append(newAds, a)
+ }
+
+ return newAds, nil
+
+}
diff --git a/node/impl/boost.go b/node/impl/boost.go
index 645fb0e49..c27be73ff 100644
--- a/node/impl/boost.go
+++ b/node/impl/boost.go
@@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error {
func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) {
return sm.StorageProvider.GetAsk(), nil
}
+
+func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) {
+ return sm.IndexProvider.AnnounceRemoveAll(ctx)
+}
diff --git a/react/src/Ipni.js b/react/src/Ipni.js
index cd7bc782a..d2b5b610b 100644
--- a/react/src/Ipni.js
+++ b/react/src/Ipni.js
@@ -111,13 +111,14 @@ function ProviderIpniInfoRender(props){
({moment(data.LastAdvertisementTime).fromNow()} ago)
- {distance.data ? ({distance.data.ipniDistanceFromLatestAd} behind): ''}
+ {distance.data ?
+ ({distance.data.ipniDistanceFromLatestAd} behind) : ''}
| Latest Advertisement on Boost |
- {lad ? {lad}: ''}
+ {lad ? {lad} : ''}
|
@@ -137,7 +138,7 @@ function ProviderConfig({configJson}) {
const cfg = JSON.parse(configJson)
return
Index Provider Config
-
+
}