storage incentives

.github/workflows/beekeeper.yml

@@ -60,7 +60,7 @@ jobs:
         run: |
           echo -e "127.0.0.10\tregistry.localhost" | sudo tee -a /etc/hosts
           for ((i=0; i<REPLICA; i++)); do echo -e "127.0.1.$((i+1))\tbee-${i}.localhost bee-${i}-debug.localhost"; done | sudo tee -a /etc/hosts
-          timeout 30m ./beeinfra.sh install --local -r "${REPLICA}" --bootnode /dnsaddr/localhost --geth --k3s --pay-threshold 1000000000000
+          timeout 30m ./beeinfra.sh install --local -r "${REPLICA}" --bootnode /dnsaddr/localhost --geth --k3s --pay-threshold 1000000000000 --postage
       - name: Test pingpong
         id: pingpong-1
         run: until ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"; do echo "waiting for pingpong..."; sleep .3; done
@@ -81,7 +81,7 @@ jobs:
         run: ./beekeeper check retrieval --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3
       - name: Test gc
         id: gc-chunk-1
-        run: ./beekeeper check gc --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --wait 15
+        run: ./beekeeper check gc --db-capacity 2000 --reserve --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --wait 15
       - name: Test manifest
         id: manifest-1
         run: ./beekeeper check manifest --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
@@ -103,7 +103,7 @@ jobs:
           cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
       - name: Set testing cluster (Node connection and clef enabled)
         run: |
-          timeout 30m ./beeinfra.sh install --local -r "${REPLICA}" --geth --clef --k3s --pay-threshold 1000000000000
+          timeout 30m ./beeinfra.sh install --local -r "${REPLICA}" --geth --clef --k3s --pay-threshold 1000000000000 --postage
       - name: Test pingpong
         id: pingpong-2
         run: until ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"; do echo "waiting for pingpong..."; sleep .3; done

cmd/bee/cmd/cmd.go

@@ -59,6 +59,8 @@ const (
 	optionNameSwapInitialDeposit        = "swap-initial-deposit"
 	optionNameSwapEnable                = "swap-enable"
 	optionNameFullNode                  = "full-node"
+	optionNamePostageContractAddress    = "postage-stamp-address"
+	optionNamePriceOracleAddress        = "price-oracle-address"
 )
 
 func init() {
@@ -226,6 +228,8 @@ func (c *command) setAllFlags(cmd *cobra.Command) {
 	cmd.Flags().String(optionNameSwapInitialDeposit, "100000000000000000", "initial deposit if deploying a new chequebook")
 	cmd.Flags().Bool(optionNameSwapEnable, true, "enable swap")
 	cmd.Flags().Bool(optionNameFullNode, true, "cause the node to start in full mode")
+	cmd.Flags().String(optionNamePostageContractAddress, "", "postage stamp contract address")
+	cmd.Flags().String(optionNamePriceOracleAddress, "", "price oracle address")
 }
 
 func newLogger(cmd *cobra.Command, verbosity string) (logging.Logger, error) {

cmd/bee/cmd/start.go

@@ -148,6 +148,8 @@ Welcome to the Swarm.... Bzzz Bzzzz Bzzzz
 				SwapInitialDeposit:       c.config.GetString(optionNameSwapInitialDeposit),
 				SwapEnable:               c.config.GetBool(optionNameSwapEnable),
 				FullNodeMode:             fullNode,
+				PostageContractAddress:   c.config.GetString(optionNamePostageContractAddress),
+				PriceOracleAddress:       c.config.GetString(optionNamePriceOracleAddress),
 			})
 			if err != nil {
 				return err

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/ethereum/go-ethereum v1.9.23
+	github.com/ethersphere/go-storage-incentives-abi v0.1.0
 	github.com/ethersphere/go-sw3-abi v0.3.2
 	github.com/ethersphere/langos v1.0.0
 	github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15

go.sum

@@ -159,6 +159,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0IdYEkbtw=
 github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM=
+github.com/ethersphere/go-storage-incentives-abi v0.1.0 h1:yxNME3q5dha/pUtIYB07DALhhQjd3+uYhGLFqKMXVyg=
+github.com/ethersphere/go-storage-incentives-abi v0.1.0/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc=
 github.com/ethersphere/go-sw3-abi v0.3.2 h1:BVTuSZ9Ph/JJBglU9pCRSch3gDq4g5QEto6KzMYP/08=
 github.com/ethersphere/go-sw3-abi v0.3.2/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU=
 github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc=
@@ -1320,6 +1322,7 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 resenje.org/daemon v0.1.2/go.mod h1:mF5JRpH3EbrxI9WoeKY78e6PqSsbBtX9jAQL5vj/GBA=
 resenje.org/email v0.1.3/go.mod h1:OhAVLRG3vqd9NSgayN3pAgzxTmc2B6mAefgShZvEgf0=
 resenje.org/jsonhttp v0.2.0/go.mod h1:EDyeguyTWj2fU3D3SCE0qNTgthzyEkHYLM1uu0uikHU=
+resenje.org/logging v0.1.5 h1:dw2TEg2kw7lhDqCCH5SqC1pFVuIFcqnTkI5PzgOhopM=
 resenje.org/logging v0.1.5/go.mod h1:1IdoCm3+UwYfsplxDGV2pHCkUrLlQzlWwp4r28XfPx4=
 resenje.org/marshal v0.1.1/go.mod h1:P7Cla6Ju5CFvW4Y8JbRgWX1Hcy4L1w4qcCsyadO7G94=
 resenje.org/recovery v0.1.1/go.mod h1:3S6aCVKMJEWsSAb61oZTteaiqkIfQPTr1RdiWnRbhME=

openapi/Swarm.yaml

@@ -705,3 +705,72 @@ paths:
           $ref: "SwarmCommon.yaml#/components/responses/500"
         default:
           description: Default response
+
+  "/stamps":
+    get:
+      summary: Get all available stamps for this node
+      tags:
+        - Get stamp batches
+      responses:
+        "200":
+          description: Returns an array of all available postage batches.
+          content:
+            application/json:
+              schema:
+                $ref: "SwarmCommon.yaml#/components/schemas/PostageBatchesResponse"
+
+        default:
+          description: Default response
+
+  "/stamps/{id}":
+    get:
+      summary: Get an individual postage batch status
+      tags:
+        - Get stamp batch
+      responses:
+        "200":
+          description: Returns an individual postage batch state
+          content:
+            application/json:
+              schema:
+                $ref: "SwarmCommon.yaml#/components/schemas/PostageBatchResponse"
+        "400":
+          $ref: "SwarmCommon.yaml#/components/responses/400"
+        default:
+          description: Default response
+
+  "/stamps/{amount}/{depth}":
+    get:
+      summary: Buy a new postage batch
+      tags:
+        - Buy stamp batch
+      parameters:
+        - in: path
+          name: amount
+          type: integer
+          required: true
+          description: Amount added to the balance
+        - in: path
+          name: depth
+          type: integer
+          required: true
+          description: Batch depth. Must be higher than default bucket depth (16)
+        - in: query
+          name: label
+          type: string
+          required: false
+          description: An optional label for this batch
+      responses:
+        "200":
+          description: Returns the newly created postage batch ID
+          content:
+            application/json:
+              schema:
+                $ref: "SwarmCommon.yaml#/components/schemas/BatchIDResponse"
+        "400":
+          $ref: "SwarmCommon.yaml#/components/responses/400"
+        "500":
+          $ref: "SwarmCommon.yaml#/components/responses/500"
+         default:
+          description: Default response
+

openapi/SwarmCommon.yaml

@@ -263,6 +263,23 @@ components:
         reference:
           $ref: "#/components/schemas/SwarmReference"
 
+    PostageBatchResponse:
+      type: object
+      properties:
+        $ref: "#/components/schemas/PostageBatch"
+
+    PostageBatchesResponse:
+      type: object
+      properties:
+        stamps:
+          $ref: "#/components/schemas/PostageBatches"
+
+    BatchIDResponse:
+      type: object
+      properties:
+        batchID:
+          $ref: "#/components/schemas/SwarmAddress"
+
     Response:
       type: object
       properties:
@@ -283,6 +300,20 @@ components:
         status:
           type: string
 
+    PostageBatch:
+      type: object
+      properties:
+        batchID:
+          $ref: "#/components/schemas/SwarmAddress"
+        utilization:
+          type: integer
+
+    PostageBatches:
+      type: array
+      items:
+        $ref: "#/components/schemas/PostageBatch"
+
+
     Settlement:
       type: object
       properties:

pkg/api/api.go

@@ -8,6 +8,7 @@ package api
 
 import (
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -19,11 +20,14 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/ethersphere/bee/pkg/crypto"
 	"github.com/ethersphere/bee/pkg/feeds"
 	"github.com/ethersphere/bee/pkg/file/pipeline/builder"
 	"github.com/ethersphere/bee/pkg/logging"
 	m "github.com/ethersphere/bee/pkg/metrics"
 	"github.com/ethersphere/bee/pkg/pinning"
+	"github.com/ethersphere/bee/pkg/postage"
+	"github.com/ethersphere/bee/pkg/postage/postagecontract"
 	"github.com/ethersphere/bee/pkg/pss"
 	"github.com/ethersphere/bee/pkg/resolver"
 	"github.com/ethersphere/bee/pkg/storage"
@@ -34,14 +38,15 @@ import (
 )
 
 const (
-	SwarmPinHeader           = "Swarm-Pin"
-	SwarmTagHeader           = "Swarm-Tag"
-	SwarmEncryptHeader       = "Swarm-Encrypt"
-	SwarmIndexDocumentHeader = "Swarm-Index-Document"
-	SwarmErrorDocumentHeader = "Swarm-Error-Document"
-	SwarmFeedIndexHeader     = "Swarm-Feed-Index"
-	SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next"
-	SwarmCollectionHeader    = "Swarm-Collection"
+	SwarmPinHeader            = "Swarm-Pin"
+	SwarmTagHeader            = "Swarm-Tag"
+	SwarmEncryptHeader        = "Swarm-Encrypt"
+	SwarmIndexDocumentHeader  = "Swarm-Index-Document"
+	SwarmErrorDocumentHeader  = "Swarm-Error-Document"
+	SwarmFeedIndexHeader      = "Swarm-Feed-Index"
+	SwarmFeedIndexNextHeader  = "Swarm-Feed-Index-Next"
+	SwarmCollectionHeader     = "Swarm-Collection"
+	SwarmPostageBatchIdHeader = "Swarm-Postage-Batch-Id"
 )
 
 // The size of buffer used for prefetching content with Langos.
@@ -65,11 +70,12 @@ const (
 var (
 	errInvalidNameOrAddress = errors.New("invalid name or bzz address")
 	errNoResolver           = errors.New("no resolver connected")
-	invalidRequest          = errors.New("could not validate request")
-	invalidContentType      = errors.New("invalid content-type")
-	invalidContentLength    = errors.New("invalid content-length")
-	directoryStoreError     = errors.New("could not store directory")
-	fileStoreError          = errors.New("could not store file")
+	errInvalidRequest       = errors.New("could not validate request")
+	errInvalidContentType   = errors.New("invalid content-type")
+	errInvalidContentLength = errors.New("invalid content-length")
+	errDirectoryStore       = errors.New("could not store directory")
+	errFileStore            = errors.New("could not store file")
+	errInvalidPostageBatch  = errors.New("invalid postage batch id")
 )
 
 // Service is the API service interface.
@@ -80,15 +86,18 @@ type Service interface {
 }
 
 type server struct {
-	tags        *tags.Tags
-	storer      storage.Storer
-	resolver    resolver.Interface
-	pss         pss.Interface
-	traversal   traversal.Traverser
-	pinning     pinning.Interface
-	logger      logging.Logger
-	tracer      *tracing.Tracer
-	feedFactory feeds.Factory
+	tags            *tags.Tags
+	storer          storage.Storer
+	resolver        resolver.Interface
+	pss             pss.Interface
+	traversal       traversal.Traverser
+	pinning         pinning.Interface
+	logger          logging.Logger
+	tracer          *tracing.Tracer
+	feedFactory     feeds.Factory
+	signer          crypto.Signer
+	post            postage.Service
+	postageContract postagecontract.Interface
 	Options
 	http.Handler
 	metrics metrics
@@ -109,20 +118,23 @@ const (
 )
 
 // New will create a and initialize a new API service.
-func New(tags *tags.Tags, storer storage.Storer, resolver resolver.Interface, pss pss.Interface, traversalService traversal.Traverser, pinning pinning.Interface, feedFactory feeds.Factory, logger logging.Logger, tracer *tracing.Tracer, o Options) Service {
+func New(tags *tags.Tags, storer storage.Storer, resolver resolver.Interface, pss pss.Interface, traversalService traversal.Traverser, pinning pinning.Interface, feedFactory feeds.Factory, post postage.Service, postageContract postagecontract.Interface, signer crypto.Signer, logger logging.Logger, tracer *tracing.Tracer, o Options) Service {
 	s := &server{
-		tags:        tags,
-		storer:      storer,
-		resolver:    resolver,
-		pss:         pss,
-		traversal:   traversalService,
-		pinning:     pinning,
-		feedFactory: feedFactory,
-		Options:     o,
-		logger:      logger,
-		tracer:      tracer,
-		metrics:     newMetrics(),
-		quit:        make(chan struct{}),
+		tags:            tags,
+		storer:          storer,
+		resolver:        resolver,
+		pss:             pss,
+		traversal:       traversalService,
+		pinning:         pinning,
+		feedFactory:     feedFactory,
+		post:            post,
+		postageContract: postageContract,
+		signer:          signer,
+		Options:         o,
+		logger:          logger,
+		tracer:          tracer,
+		metrics:         newMetrics(),
+		quit:            make(chan struct{}),
 	}
 
 	s.setupRouting()
@@ -211,6 +223,21 @@ func requestEncrypt(r *http.Request) bool {
 	return strings.ToLower(r.Header.Get(SwarmEncryptHeader)) == "true"
 }
 
+func requestPostageBatchId(r *http.Request) ([]byte, error) {
+	if h := strings.ToLower(r.Header.Get(SwarmPostageBatchIdHeader)); h != "" {
+		if len(h) != 64 {
+			return nil, errInvalidPostageBatch
+		}
+		b, err := hex.DecodeString(h)
+		if err != nil {
+			return nil, errInvalidPostageBatch
+		}
+		return b, nil
+	}
+
+	return nil, errInvalidPostageBatch
+}
+
 func (s *server) newTracingHandler(spanName string) func(h http.Handler) http.Handler {
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -285,9 +312,36 @@ func equalASCIIFold(s, t string) bool {
 	return s == t
 }
 
+type stamperPutter struct {
+	storage.Storer
+	stamper postage.Stamper
+}
+
+func newStamperPutter(s storage.Storer, post postage.Service, signer crypto.Signer, batch []byte) (storage.Storer, error) {
+	i, err := post.GetStampIssuer(batch)
+	if err != nil {
+		return nil, fmt.Errorf("stamp issuer: %w", err)
+	}
+
+	stamper := postage.NewStamper(i, signer)
+	return &stamperPutter{Storer: s, stamper: stamper}, nil
+}
+
+func (p *stamperPutter) Put(ctx context.Context, mode storage.ModePut, chs ...swarm.Chunk) (exist []bool, err error) {
+	for i, c := range chs {
+		stamp, err := p.stamper.Stamp(c.Address())
+		if err != nil {
+			return nil, err
+		}
+		chs[i] = c.WithStamp(stamp)
+	}
+
+	return p.Storer.Put(ctx, mode, chs...)
+}
+
 type pipelineFunc func(context.Context, io.Reader, int64) (swarm.Address, error)
 
-func requestPipelineFn(s storage.Storer, r *http.Request) pipelineFunc {
+func requestPipelineFn(s storage.Putter, r *http.Request) pipelineFunc {
 	mode, encrypt := requestModePut(r), requestEncrypt(r)
 	return func(ctx context.Context, r io.Reader, l int64) (swarm.Address, error) {
 		pipe := builder.NewPipelineBuilder(ctx, s, mode, encrypt)