Merge from main to lid branch (#1483)

* fix statx output string (#1451) * fix: flaky TestMultipleDealsConcurrent (#1458) * Add option to serve index provider ads over http (#1452) * feat: option to serve index provider ads over http * fix: config naming, hostname parsing * fix: update docsgen * fix: log announce address * feat: add config for indexer direct announce urls * refactor: always announce over pubsub * fix: docsgen * test: add test case for empty announce address hostname * Add `boostd index announce-latest` command (#1456) * feat: boostd index announce-latest * feat: add announce-latest-http command * fix: default direct announce url * feat: update to index-provider v0.11.2 * Signal to index provider to skip announcements (#1457) * fix: signal to index provider to skip announcements * fix: ensure multihash lister skip error is of type ipld.ErrNotExists --------- Co-authored-by: LexLuthr <[email protected]> * release v1.7.3-rc2 (#1460) * fix: improve stalled retrieval cancellation (#1449) * refactor stalled retrieval cancel * add ctx with timeout * implement suggestions * update err wrapping * fix: set short cancel timeout for unpaid retrievals only --------- Co-authored-by: Dirk McCormick <[email protected]> * feat: enable listen address for booster-http (#1461) * enable listen address * modify tests * fix nil ptr (#1470) * fix: incorrect check when import offline deal data using proposal CID (#1473) * fix incorrect early check * update error msg * fix(server): properly cancel graphsync requests (#1475) * set UI default listen address to localhost (#1476) * feat: display msg params in the mpool UI (#1471) * show msg params * fix: mpool nil pointer * fix width --------- Co-authored-by: Dirk McCormick <[email protected]> * Reset read deadline after reading deal proposal message (#1479) * fix: reset read deadline after reading deal proposal message * fix: increase client request deadline * feat: Show elapsed epoch and PSD wait epochs in UI (#1480) * show epochs * fix devnet UI, use BlockdDelaySecs * fix lint err * Update gql/resolver.go Co-authored-by: dirkmc <[email protected]> --------- Co-authored-by: dirkmc <[email protected]> * release v1.7.3-rc3 (#1481) --------- Co-authored-by: LexLuthr <[email protected]> Co-authored-by: LexLuthr <[email protected]> Co-authored-by: Hannah Howard <[email protected]>
filecoin-project · Jun 1, 2023 · 5e511a8 · 5e511a8
1 parent 5fe0bfd
commit 5e511a8
Show file tree

Hide file tree

Showing 33 changed files with 704 additions and 200 deletions.
diff --git a/api/api.go b/api/api.go
@@ -35,6 +35,8 @@ type Boost interface {
 	// MethodGroup: Boost
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error)                                        //perm:admin
+	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin
+	BoostIndexerAnnounceLatestHttp(ctx context.Context, urls []string) (cid.Cid, error)                                                         //perm:admin
 	BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin
 	BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error)                                                      //perm:admin
 	BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error)                                  //perm:admin

diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/build/version.go b/build/version.go
@@ -2,7 +2,7 @@ package build
 
 var CurrentCommit string
 
-const BuildVersion = "1.7.3-rc1"
+const BuildVersion = "1.7.3-rc3"
 
 func UserVersion() string {
 	return BuildVersion + CurrentCommit

diff --git a/cmd/boostd/import_data.go b/cmd/boostd/import_data.go
@@ -67,9 +67,6 @@ var importDataCmd = &cli.Command{
 		// If the user has supplied a signed proposal cid
 		deleteAfterImport := cctx.Bool("delete-after-import")
 		if proposalCid != nil {
-			if deleteAfterImport {
-				return fmt.Errorf("legacy deal data cannot be automatically deleted after import (only new deals)")
-			}
 
 			// Look up the deal in the boost database
 			deal, err := napi.BoostDealBySignedProposalCid(cctx.Context, *proposalCid)
@@ -80,6 +77,10 @@ var importDataCmd = &cli.Command{
 					return err
 				}
 
+				if deleteAfterImport {
+					return fmt.Errorf("cannot find boost deal with proposal cid %s and legacy deal data cannot be automatically deleted after import (only new deals)", proposalCid)
+				}
+
 				// The deal is not in the boost database, try the legacy
 				// markets datastore (v1.1.0 deal)
 				err := napi.MarketImportDealData(cctx.Context, *proposalCid, filePath)

diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go
@@ -14,6 +14,8 @@ var indexProvCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		indexProvAnnounceAllCmd,
 		indexProvListMultihashesCmd,
+		indexProvAnnounceLatest,
+		indexProvAnnounceLatestHttp,
 	},
 }
 
@@ -64,11 +66,63 @@ var indexProvListMultihashesCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-
+		
 		fmt.Printf("Found %d multihashes for deal with proposal cid %s:\n", len(mhs), propCid)
 		for _, mh := range mhs {
 			fmt.Println("  " + mh.String())
 		}
+
+		return nil
+	},
+}
+
+var indexProvAnnounceLatest = &cli.Command{
+	Name:  "announce-latest",
+	Usage: "Re-publish the latest existing advertisement to pubsub",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		c, err := napi.BoostIndexerAnnounceLatest(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Announced advertisement with cid %s\n", c)
+		return nil
+	},
+}
+
+var indexProvAnnounceLatestHttp = &cli.Command{
+	Name:  "announce-latest-http",
+	Usage: "Re-publish the latest existing advertisement to specific indexers over http",
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "announce-url",
+			Usage:    "The url(s) to announce to. If not specified, announces to the http urls in config",
+			Required: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		c, err := napi.BoostIndexerAnnounceLatestHttp(ctx, cctx.StringSlice("announce-url"))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Announced advertisement to indexers over http with cid %s\n", c)
 		return nil
 	},
 }
diff --git a/cmd/boostd/init.go b/cmd/boostd/init.go
@@ -493,7 +493,13 @@ func migrateMarketsConfig(cctx *cli.Context, mktsRepo lotus_repo.LockedRepo, boo
 		// Clear the DAG store root dir config, because the DAG store is no longer configurable in Boost
 		// (it is always at <repo path>/dagstore
 		rcfg.DAGStore.RootDir = ""
-		rcfg.IndexProvider = mktsCfg.IndexProvider
+		rcfg.IndexProvider = config.IndexProviderConfig{
+			Enable:               mktsCfg.IndexProvider.Enable,
+			EntriesCacheCapacity: mktsCfg.IndexProvider.EntriesCacheCapacity,
+			EntriesChunkSize:     mktsCfg.IndexProvider.EntriesChunkSize,
+			TopicName:            mktsCfg.IndexProvider.TopicName,
+			PurgeCacheOnStart:    mktsCfg.IndexProvider.PurgeCacheOnStart,
+		}
 		rcfg.IndexProvider.Enable = true // Enable index provider in Boost by default
 
 		if fromMonolith {

diff --git a/cmd/booster-http/http_test.go b/cmd/booster-http/http_test.go
@@ -22,7 +22,7 @@ const testFile = "test/test_file"
 func TestNewHttpServer(t *testing.T) {
 	// Create a new mock Http server
 	ctrl := gomock.NewController(t)
-	httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
+	httpServer := NewHttpServer("", "0.0.0.0", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
 	err := httpServer.Start(context.Background())
 	require.NoError(t, err)
 	waitServerUp(t, 7777)
@@ -41,7 +41,7 @@ func TestHttpGzipResponse(t *testing.T) {
 	// Create a new mock Http server with custom functions
 	ctrl := gomock.NewController(t)
 	mockHttpServer := mocks_booster_http.NewMockHttpServerApi(ctrl)
-	httpServer := NewHttpServer("", 7777, mockHttpServer, nil)
+	httpServer := NewHttpServer("", "0.0.0.0", 7777, mockHttpServer, nil)
 	err := httpServer.Start(context.Background())
 	require.NoError(t, err)
 	waitServerUp(t, 7777)
@@ -99,7 +99,7 @@ func TestHttpInfo(t *testing.T) {
 
 	// Create a new mock Http server
 	ctrl := gomock.NewController(t)
-	httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
+	httpServer := NewHttpServer("", "0.0.0.0", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil)
 	err := httpServer.Start(context.Background())
 	require.NoError(t, err)
 	waitServerUp(t, 7777)

diff --git a/cmd/booster-http/mocks/mock_booster_http.go b/cmd/booster-http/mocks/mock_booster_http.go
diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go
@@ -40,6 +40,12 @@ var runCmd = &cli.Command{
 			Usage: "the base path at which to run the web server",
 			Value: "",
 		},
+		&cli.StringFlag{
+			Name:    "address",
+			Aliases: []string{"addr"},
+			Usage:   "the listen address for the web server",
+			Value:   "0.0.0.0",
+		},
 		&cli.UintFlag{
 			Name:  "port",
 			Usage: "the port the web server listens on",
@@ -207,6 +213,7 @@ var runCmd = &cli.Command{
 		sapi := serverApi{ctx: ctx, piecedirectory: pd, sa: sa}
 		server := NewHttpServer(
 			cctx.String("base-path"),
+			cctx.String("address"),
 			cctx.Int("port"),
 			sapi,
 			opts,
@@ -216,8 +223,8 @@ var runCmd = &cli.Command{
 		pd.Start(ctx)
 
 		// Start the server
-		log.Infof("Starting booster-http node on port %d with base path '%s'",
-			cctx.Int("port"), cctx.String("base-path"))
+		log.Infof("Starting booster-http node on listen address %s and port %d with base path '%s'",
+			cctx.String("address"), cctx.Int("port"), cctx.String("base-path"))
 		err = server.Start(ctx)
 		if err != nil {
 			return fmt.Errorf("starting http server: %w", err)

diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go
@@ -42,11 +42,12 @@ type apiVersion struct {
 }
 
 type HttpServer struct {
-	path    string
-	port    int
-	api     HttpServerApi
-	opts    HttpServerOptions
-	idxPage string
+	path       string
+	listenAddr string
+	port       int
+	api        HttpServerApi
+	opts       HttpServerOptions
+	idxPage    string
 
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -65,11 +66,11 @@ type HttpServerOptions struct {
 	SupportedResponseFormats []string
 }
 
-func NewHttpServer(path string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer {
+func NewHttpServer(path string, listenAddr string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer {
 	if opts == nil {
 		opts = &HttpServerOptions{ServePieces: true}
 	}
-	return &HttpServer{path: path, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)}
+	return &HttpServer{path: path, listenAddr: listenAddr, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)}
 }
 
 func (s *HttpServer) pieceBasePath() string {
@@ -102,7 +103,7 @@ func (s *HttpServer) Start(ctx context.Context) error {
 	handler.HandleFunc("/info", s.handleInfo)
 	handler.Handle("/metrics", metrics.Exporter("booster_http")) // metrics
 	s.server = &http.Server{
-		Addr:    fmt.Sprintf(":%d", s.port),
+		Addr:    fmt.Sprintf("%s:%d", s.listenAddr, s.port),
 		Handler: handler,
 		// This context will be the parent of the context associated with all
 		// incoming requests

diff --git a/cmd/boostx/stats_cmd.go b/cmd/boostx/stats_cmd.go
@@ -2,13 +2,14 @@ package main
 
 import (
 	"fmt"
-	"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
-	transports_types "github.com/filecoin-project/boost/retrievalmarket/types"
 	"regexp"
 	"sort"
 	"strings"
 	"sync"
 
+	"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
+	transports_types "github.com/filecoin-project/boost/retrievalmarket/types"
+
 	clinode "github.com/filecoin-project/boost/cli/node"
 	"github.com/filecoin-project/boost/cmd"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
@@ -223,7 +224,7 @@ var statsCmd = &cli.Command{
 		fmt.Println("Total Boost nodes:", boostNodes)
 		fmt.Println("Total Boost raw power:", boostRawBytePower)
 		fmt.Println("Total Boost quality adj power:", boostQualityAdjPower)
-		fmt.Println("Total Lotus Markets nodes:", marketsNodes)
+		fmt.Println("Total Markets nodes:", marketsNodes)
 		fmt.Println("Total SPs with minimum power: ", len(withMinPower))
 		fmt.Println("Total Indexer nodes:", indexerNodes)
 

diff --git a/docker/devnet/boost/entrypoint.sh b/docker/devnet/boost/entrypoint.sh
@@ -40,6 +40,7 @@ if [ ! -f $BOOST_PATH/.init.boost ]; then
 
 	echo Setting port in boost config...
 	sed -i 's|ip4/0.0.0.0/tcp/0|ip4/0.0.0.0/tcp/50000|g' $BOOST_PATH/config.toml
+	sed -i 's|127.0.0.1|0.0.0.0|g' $BOOST_PATH/config.toml
 
 	echo Done
 	touch $BOOST_PATH/.init.boost

diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
@@ -23,6 +23,8 @@
   * [BoostDealBySignedProposalCid](#boostdealbysignedproposalcid)
   * [BoostDummyDeal](#boostdummydeal)
   * [BoostIndexerAnnounceAllDeals](#boostindexerannouncealldeals)
+  * [BoostIndexerAnnounceLatest](#boostindexerannouncelatest)
+  * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
   * [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
   * [BoostMakeDeal](#boostmakedeal)
   * [BoostOfflineDealWithData](#boostofflinedealwithdata)
@@ -560,6 +562,41 @@ Inputs: `null`
 
 Response: `{}`
 
+### BoostIndexerAnnounceLatest
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
+### BoostIndexerAnnounceLatestHttp
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "string value"
+  ]
+]
+```
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
 ### BoostIndexerListMultihashes
 
 

diff --git a/go.mod b/go.mod
@@ -76,7 +76,7 @@ require (
 	github.com/ipld/go-car/v2 v2.7.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
-	github.com/ipni/index-provider v0.11.1
+	github.com/ipni/index-provider v0.11.2
 	github.com/ipni/storetheindex v0.5.10
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jpillora/backoff v1.0.0

diff --git a/go.sum b/go.sum
@@ -981,8 +981,8 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
-github.com/ipni/index-provider v0.11.1 h1:viNfSBvZA9G+Qe6/FGqfZtavnu4tTSfGUoWEECavqoI=
-github.com/ipni/index-provider v0.11.1/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
+github.com/ipni/index-provider v0.11.2 h1:nvykWK+/ncPTqHiuiJdXp/O0UF0V7iWesjHGKX//NYc=
+github.com/ipni/index-provider v0.11.2/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
 github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4=
 github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=