storage-incentives: cleanups (#1602)

* deprecate empty batches fallback * adjust bucketDepth and default radius * fix long standing rebase error in node.go some renaming improvments too
ethersphere · Apr 27, 2021 · 09dad20 · 09dad20
1 parent 6a49005
commit 09dad20
Show file tree

Hide file tree

Showing 24 changed files with 203 additions and 90 deletions.
diff --git a/pkg/api/api.go b/pkg/api/api.go
@@ -235,8 +235,7 @@ func requestPostageBatchId(r *http.Request) ([]byte, error) {
 		return b, nil
 	}
 
-	// fallback to a slice of 32 zeros
-	return make([]byte, 32), nil
+	return nil, errInvalidPostageBatch
 }
 
 func (s *server) newTracingHandler(spanName string) func(h http.Handler) http.Handler {

diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go
@@ -6,6 +6,7 @@ package api_test
 
 import (
 	"bytes"
+	"crypto/rand"
 	"encoding/hex"
 	"errors"
 	"io"
@@ -22,6 +23,7 @@ import (
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/pinning"
+	"github.com/ethersphere/bee/pkg/postage"
 	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
 	"github.com/ethersphere/bee/pkg/postage/postagecontract"
 	"github.com/ethersphere/bee/pkg/pss"
@@ -39,10 +41,17 @@ import (
 
 var (
 	batchInvalid = []byte{0}
-	batchOk      = []byte{31: 0} // 32 zeros
+	batchOk      = make([]byte, 32)
+	batchOkStr   string
 	batchEmpty   = []byte{}
 )
 
+func init() {
+	_, _ = rand.Read(batchOk)
+
+	batchOkStr = hex.EncodeToString(batchOk)
+}
+
 type testServerOptions struct {
 	Storer             storage.Storer
 	Resolver           resolver.Interface
@@ -58,13 +67,13 @@ type testServerOptions struct {
 	Feeds              feeds.Factory
 	CORSAllowedOrigins []string
 	PostageContract    postagecontract.Interface
+	Post               postage.Service
 }
 
 func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket.Conn, string) {
 	t.Helper()
 	pk, _ := crypto.GenerateSecp256k1Key()
 	signer := crypto.NewDefaultSigner(pk)
-	mockPostage := mockpost.New()
 
 	if o.Logger == nil {
 		o.Logger = logging.New(ioutil.Discard, 0)
@@ -75,7 +84,10 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket.
 	if o.WsPingPeriod == 0 {
 		o.WsPingPeriod = 60 * time.Second
 	}
-	s := api.New(o.Tags, o.Storer, o.Resolver, o.Pss, o.Traversal, o.Pinning, o.Feeds, mockPostage, o.PostageContract, signer, o.Logger, nil, api.Options{
+	if o.Post == nil {
+		o.Post = mockpost.New()
+	}
+	s := api.New(o.Tags, o.Storer, o.Resolver, o.Pss, o.Traversal, o.Pinning, o.Feeds, o.Post, o.PostageContract, signer, o.Logger, nil, api.Options{
 		CORSAllowedOrigins: o.CORSAllowedOrigins,
 		GatewayMode:        o.GatewayMode,
 		WsPingPeriod:       o.WsPingPeriod,
@@ -253,10 +265,12 @@ func TestPostageHeaderError(t *testing.T) {
 		mockStorer     = mock.NewStorer()
 		mockStatestore = statestore.NewStateStore()
 		logger         = logging.New(ioutil.Discard, 5)
+		mp             = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10)))
 		client, _, _   = newTestServer(t, testServerOptions{
 			Storer: mockStorer,
 			Tags:   tags.NewTags(mockStatestore, logger),
 			Logger: logger,
+			Post:   mp,
 		})
 
 		endpoints = []string{
@@ -267,14 +281,14 @@ func TestPostageHeaderError(t *testing.T) {
 	for _, endpoint := range endpoints {
 		t.Run(endpoint+": empty batch", func(t *testing.T) {
 			hexbatch := hex.EncodeToString(batchEmpty)
-			expCode := http.StatusOK
+			expCode := http.StatusBadRequest
 			jsonhttptest.Request(t, client, http.MethodPost, "/"+endpoint, expCode,
 				jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, hexbatch),
 				jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "application/octet-stream"),
 				jsonhttptest.WithRequestBody(bytes.NewReader(content)),
 			)
 		})
-		t.Run(endpoint+": all zeros - ok", func(t *testing.T) {
+		t.Run(endpoint+": ok batch", func(t *testing.T) {
 			hexbatch := hex.EncodeToString(batchOk)
 			expCode := http.StatusOK
 			jsonhttptest.Request(t, client, http.MethodPost, "/"+endpoint, expCode,

diff --git a/pkg/api/bytes_test.go b/pkg/api/bytes_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/logging"
 	pinning "github.com/ethersphere/bee/pkg/pinning/mock"
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
 	statestore "github.com/ethersphere/bee/pkg/statestore/mock"
 	"github.com/ethersphere/bee/pkg/storage/mock"
 	"github.com/ethersphere/bee/pkg/swarm"
@@ -41,6 +42,7 @@ func TestBytes(t *testing.T) {
 			Tags:    tags.NewTags(statestore.NewStateStore(), logging.New(ioutil.Discard, 0)),
 			Pinning: pinningMock,
 			Logger:  logger,
+			Post:    mockpost.New(mockpost.WithAcceptAll()),
 		})
 	)
 
@@ -53,6 +55,7 @@ func TestBytes(t *testing.T) {
 	t.Run("upload", func(t *testing.T) {
 		chunkAddr := swarm.MustParseHexAddress(expHash)
 		jsonhttptest.Request(t, client, http.MethodPost, resource, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(content)),
 			jsonhttptest.WithExpectedJSONResponse(api.BytesPostResponse{
 				Reference: chunkAddr,
@@ -75,6 +78,7 @@ func TestBytes(t *testing.T) {
 	t.Run("upload-with-pinning", func(t *testing.T) {
 		var res api.BytesPostResponse
 		jsonhttptest.Request(t, client, http.MethodPost, resource, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(content)),
 			jsonhttptest.WithRequestHeader(api.SwarmPinHeader, "true"),
 			jsonhttptest.WithUnmarshalJSONResponse(&res),

diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/manifest"
 	pinning "github.com/ethersphere/bee/pkg/pinning/mock"
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
 	statestore "github.com/ethersphere/bee/pkg/statestore/mock"
 	"github.com/ethersphere/bee/pkg/storage"
 	smock "github.com/ethersphere/bee/pkg/storage/mock"
@@ -46,12 +47,14 @@ func TestBzzFiles(t *testing.T) {
 			Pinning: pinningMock,
 			Tags:    tags.NewTags(statestoreMock, logger),
 			Logger:  logger,
+			Post:    mockpost.New(mockpost.WithAcceptAll()),
 		})
 	)
 
 	t.Run("invalid-content-type", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource,
 			http.StatusBadRequest,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
 				Message: api.InvalidContentType.Error(),
@@ -89,6 +92,7 @@ func TestBzzFiles(t *testing.T) {
 		})
 		address := swarm.MustParseHexAddress("f30c0aa7e9e2a0ef4c9b1b750ebfeaeb7c7c24da700bb089da19a46e3677824b")
 		jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(tr),
 			jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
 			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
@@ -138,6 +142,7 @@ func TestBzzFiles(t *testing.T) {
 		})
 		address := swarm.MustParseHexAddress("f30c0aa7e9e2a0ef4c9b1b750ebfeaeb7c7c24da700bb089da19a46e3677824b")
 		jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestHeader(api.SwarmPinHeader, "true"),
 			jsonhttptest.WithRequestBody(tr),
 			jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
@@ -172,6 +177,7 @@ func TestBzzFiles(t *testing.T) {
 		var resp api.BzzUploadResponse
 		jsonhttptest.Request(t, client, http.MethodPost,
 			fileUploadResource+"?name="+fileName, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
 			jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, "True"),
 			jsonhttptest.WithRequestHeader("Content-Type", "image/jpeg; charset=utf-8"),
@@ -202,6 +208,7 @@ func TestBzzFiles(t *testing.T) {
 
 		jsonhttptest.Request(t, client, http.MethodPost,
 			fileUploadResource+"?name="+fileName, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
 			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
@@ -242,6 +249,7 @@ func TestBzzFiles(t *testing.T) {
 
 		rcvdHeader := jsonhttptest.Request(t, client, http.MethodPost,
 			fileUploadResource+"?name="+fileName, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(strings.NewReader(sampleHtml)),
 			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
@@ -280,6 +288,7 @@ func TestBzzFiles(t *testing.T) {
 
 		jsonhttptest.Request(t, client, http.MethodPost,
 			fileUploadResource+"?name="+fileName, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
 			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
@@ -392,11 +401,13 @@ func TestBzzFilesRangeRequests(t *testing.T) {
 				Storer: smock.NewStorer(),
 				Tags:   tags.NewTags(mockStatestore, logger),
 				Logger: logger,
+				Post:   mockpost.New(mockpost.WithAcceptAll()),
 			})
 
 			var resp api.BzzUploadResponse
 
 			testOpts := []jsonhttptest.Option{
+				jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 				jsonhttptest.WithRequestBody(upload.reader),
 				jsonhttptest.WithRequestHeader("Content-Type", upload.contentType),
 				jsonhttptest.WithUnmarshalJSONResponse(&resp),
@@ -505,6 +516,7 @@ func TestFeedIndirection(t *testing.T) {
 			Storer: storer,
 			Tags:   tags.NewTags(mockStatestore, logger),
 			Logger: logger,
+			Post:   mockpost.New(mockpost.WithAcceptAll()),
 		})
 	)
 	// tar all the test case files
@@ -520,6 +532,7 @@ func TestFeedIndirection(t *testing.T) {
 	var resp api.BzzUploadResponse
 
 	options := []jsonhttptest.Option{
+		jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 		jsonhttptest.WithRequestBody(tarReader),
 		jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
 		jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),

diff --git a/pkg/api/chunk_test.go b/pkg/api/chunk_test.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/ethersphere/bee/pkg/logging"
 	pinning "github.com/ethersphere/bee/pkg/pinning/mock"
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
 	statestore "github.com/ethersphere/bee/pkg/statestore/mock"
 
 	"github.com/ethersphere/bee/pkg/tags"
@@ -45,11 +46,13 @@ func TestChunkUploadDownload(t *testing.T) {
 			Storer:  storerMock,
 			Pinning: pinningMock,
 			Tags:    tag,
+			Post:    mockpost.New(mockpost.WithAcceptAll()),
 		})
 	)
 
 	t.Run("empty chunk", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, chunksEndpoint, http.StatusBadRequest,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
 				Message: "data length",
 				Code:    http.StatusBadRequest,
@@ -59,6 +62,7 @@ func TestChunkUploadDownload(t *testing.T) {
 
 	t.Run("ok", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, chunksEndpoint, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(chunk.Data())),
 			jsonhttptest.WithExpectedJSONResponse(api.ChunkAddressResponse{Reference: chunk.Address()}),
 		)
@@ -77,6 +81,7 @@ func TestChunkUploadDownload(t *testing.T) {
 
 	t.Run("pin-invalid-value", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, chunksEndpoint, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(chunk.Data())),
 			jsonhttptest.WithExpectedJSONResponse(api.ChunkAddressResponse{Reference: chunk.Address()}),
 			jsonhttptest.WithRequestHeader(api.SwarmPinHeader, "invalid-pin"),
@@ -89,6 +94,7 @@ func TestChunkUploadDownload(t *testing.T) {
 	})
 	t.Run("pin-header-missing", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, chunksEndpoint, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(chunk.Data())),
 			jsonhttptest.WithExpectedJSONResponse(api.ChunkAddressResponse{Reference: chunk.Address()}),
 		)
@@ -101,6 +107,7 @@ func TestChunkUploadDownload(t *testing.T) {
 	t.Run("pin-ok", func(t *testing.T) {
 		address := chunk.Address()
 		jsonhttptest.Request(t, client, http.MethodPost, chunksEndpoint, http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(chunk.Data())),
 			jsonhttptest.WithExpectedJSONResponse(api.ChunkAddressResponse{Reference: address}),
 			jsonhttptest.WithRequestHeader(api.SwarmPinHeader, "True"),

diff --git a/pkg/api/dirs_test.go b/pkg/api/dirs_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/manifest"
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
 	statestore "github.com/ethersphere/bee/pkg/statestore/mock"
 	"github.com/ethersphere/bee/pkg/storage"
 	"github.com/ethersphere/bee/pkg/storage/mock"
@@ -44,12 +45,14 @@ func TestDirs(t *testing.T) {
 			Tags:            tags.NewTags(mockStatestore, logger),
 			Logger:          logger,
 			PreventRedirect: true,
+			Post:            mockpost.New(mockpost.WithAcceptAll()),
 		})
 	)
 
 	t.Run("empty request body", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource,
 			http.StatusBadRequest,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(bytes.NewReader(nil)),
 			jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
@@ -65,6 +68,7 @@ func TestDirs(t *testing.T) {
 
 		jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource,
 			http.StatusInternalServerError,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(file),
 			jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
@@ -84,6 +88,7 @@ func TestDirs(t *testing.T) {
 		// submit valid tar, but with wrong content-type
 		jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource,
 			http.StatusBadRequest,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithRequestBody(tarReader),
 			jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
@@ -382,6 +387,7 @@ func TestDirs(t *testing.T) {
 				var resp api.BzzUploadResponse
 
 				options := []jsonhttptest.Option{
+					jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 					jsonhttptest.WithRequestBody(tarReader),
 					jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
 					jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
@@ -414,6 +420,7 @@ func TestDirs(t *testing.T) {
 					var resp api.BzzUploadResponse
 
 					options := []jsonhttptest.Option{
+						jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 						jsonhttptest.WithRequestBody(mwReader),
 						jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
 						jsonhttptest.WithRequestHeader("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mwBoundary)),

diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go
@@ -22,6 +22,8 @@ import (
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/manifest"
+	"github.com/ethersphere/bee/pkg/postage"
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
 	testingsoc "github.com/ethersphere/bee/pkg/soc/testing"
 	statestore "github.com/ethersphere/bee/pkg/statestore/mock"
 	"github.com/ethersphere/bee/pkg/storage"
@@ -152,17 +154,20 @@ func TestFeed_Post(t *testing.T) {
 		logger         = logging.New(ioutil.Discard, 0)
 		tag            = tags.NewTags(mockStatestore, logger)
 		topic          = "aabbcc"
+		mp             = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10)))
 		mockStorer     = mock.NewStorer()
 		client, _, _   = newTestServer(t, testServerOptions{
 			Storer: mockStorer,
 			Tags:   tag,
 			Logger: logger,
+			Post:   mp,
 		})
 		url = fmt.Sprintf("/feeds/%s/%s?type=%s", ownerString, topic, "sequence")
 	)
 
 	t.Run("ok", func(t *testing.T) {
 		jsonhttptest.Request(t, client, http.MethodPost, url, http.StatusCreated,
+			jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr),
 			jsonhttptest.WithExpectedJSONResponse(api.FeedReferenceResponse{
 				Reference: expReference,
 			}),
@@ -206,9 +211,9 @@ func TestFeed_Post(t *testing.T) {
 				jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, hexbatch),
 			)
 		})
-		t.Run("ok - batch empty", func(t *testing.T) {
+		t.Run("bad request - batch empty", func(t *testing.T) {
 			hexbatch := hex.EncodeToString(batchEmpty)
-			jsonhttptest.Request(t, client, http.MethodPost, url, http.StatusCreated,
+			jsonhttptest.Request(t, client, http.MethodPost, url, http.StatusBadRequest,
 				jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, hexbatch),
 			)
 		})