feat(taiko-client): integrate SocialScan blob storage service (#17297)

packages/taiko-client/cmd/flags/driver.go

@@ -46,6 +46,12 @@ var (
 		Category: driverCategory,
 		EnvVars:  []string{"BLOB_SERVER"},
 	}
+	SocialScanEndpoint = &cli.StringFlag{
+		Name:     "blob.socialScanEndpoint",
+		Usage:    "Social Scan's blob storage server",
+		Category: driverCategory,
+		EnvVars:  []string{"BLOB_SOCIAL_SCAN_ENDPOINT"},
+	}
 )
 
 // DriverFlags All driver flags.
@@ -59,4 +65,5 @@ var DriverFlags = MergeFlags(CommonFlags, []cli.Flag{
 	CheckPointSyncURL,
 	MaxExponent,
 	BlobServerEndpoint,
+	SocialScanEndpoint,
 })

packages/taiko-client/driver/chain_syncer/blob/syncer.go

@@ -56,6 +56,7 @@ func NewSyncer(
 	progressTracker *beaconsync.SyncProgressTracker,
 	maxRetrieveExponent uint64,
 	blobServerEndpoint *url.URL,
+	socialScanEndpoint *url.URL,
 ) (*Syncer, error) {
 	configs, err := client.TaikoL1.GetConfig(&bind.CallOpts{Context: ctx})
 	if err != nil {
@@ -83,6 +84,7 @@ func NewSyncer(
 			ctx,
 			client,
 			blobServerEndpoint,
+			socialScanEndpoint,
 		),
 	}, nil
 }

packages/taiko-client/driver/chain_syncer/blob/syncer_test.go

@@ -42,6 +42,7 @@ func (s *BlobSyncerTestSuite) SetupTest() {
 		beaconsync.NewSyncProgressTracker(s.RPCClient.L2, 1*time.Hour),
 		0,
 		nil,
+		nil,
 	)
 	s.Nil(err)
 	s.s = syncer
@@ -58,6 +59,7 @@ func (s *BlobSyncerTestSuite) TestCancelNewSyncer() {
 		s.s.progressTracker,
 		0,
 		nil,
+		nil,
 	)
 	s.Nil(syncer)
 	s.NotNil(err)

packages/taiko-client/driver/chain_syncer/chain_syncer.go

@@ -47,7 +47,7 @@ func New(
 	p2pSyncTimeout time.Duration,
 	maxRetrieveExponent uint64,
 	blobServerEndpoint *url.URL,
-
+	socialScanEndpoint *url.URL,
 ) (*L2ChainSyncer, error) {
 	tracker := beaconsync.NewSyncProgressTracker(rpc.L2, p2pSyncTimeout)
 	go tracker.Track(ctx)
@@ -57,7 +57,15 @@ func New(
 		return nil, err
 	}
 	beaconSyncer := beaconsync.NewSyncer(ctx, rpc, state, syncMode, tracker)
-	blobSyncer, err := blob.NewSyncer(ctx, rpc, state, tracker, maxRetrieveExponent, blobServerEndpoint)
+	blobSyncer, err := blob.NewSyncer(
+		ctx,
+		rpc,
+		state,
+		tracker,
+		maxRetrieveExponent,
+		blobServerEndpoint,
+		socialScanEndpoint,
+	)
 	if err != nil {
 		return nil, err
 	}

packages/taiko-client/driver/chain_syncer/chain_syncer_test.go

@@ -42,6 +42,7 @@ func (s *ChainSyncerTestSuite) SetupTest() {
 		1*time.Hour,
 		0,
 		nil,
+		nil,
 	)
 	s.Nil(err)
 	s.s = syncer

packages/taiko-client/driver/config.go

@@ -22,6 +22,7 @@ type Config struct {
 	RetryInterval      time.Duration
 	MaxExponent        uint64
 	BlobServerEndpoint *url.URL
+	SocialScanEndpoint *url.URL
 }
 
 // NewConfigFromCliContext creates a new config instance from
@@ -54,6 +55,15 @@ func NewConfigFromCliContext(c *cli.Context) (*Config, error) {
 		}
 	}
 
+	var socialScanEndpoint *url.URL
+	if c.IsSet(flags.SocialScanEndpoint.Name) {
+		if socialScanEndpoint, err = url.Parse(
+			c.String(flags.SocialScanEndpoint.Name),
+		); err != nil {
+			return nil, err
+		}
+	}
+
 	var timeout = c.Duration(flags.RPCTimeout.Name)
 	return &Config{
 		ClientConfig: &rpc.ClientConfig{
@@ -72,5 +82,6 @@ func NewConfigFromCliContext(c *cli.Context) (*Config, error) {
 		P2PSyncTimeout:     c.Duration(flags.P2PSyncTimeout.Name),
 		MaxExponent:        c.Uint64(flags.MaxExponent.Name),
 		BlobServerEndpoint: blobServerEndpoint,
+		SocialScanEndpoint: socialScanEndpoint,
 	}, nil
 }

packages/taiko-client/driver/driver.go

@@ -81,6 +81,7 @@ func (d *Driver) InitFromConfig(ctx context.Context, cfg *Config) (err error) {
 		cfg.P2PSyncTimeout,
 		cfg.MaxExponent,
 		cfg.BlobServerEndpoint,
+		cfg.SocialScanEndpoint,
 	); err != nil {
 		return err
 	}

packages/taiko-client/pkg/rpc/blob_datasource.go

@@ -2,6 +2,7 @@ package rpc
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/url"
 
@@ -18,6 +19,7 @@ type BlobDataSource struct {
 	ctx                context.Context
 	client             *Client
 	blobServerEndpoint *url.URL
+	socialScanEndpoint *url.URL
 }
 
 type BlobData struct {
@@ -40,14 +42,36 @@ func NewBlobDataSource(
 	ctx context.Context,
 	client *Client,
 	blobServerEndpoint *url.URL,
+	socialScanEndpoint *url.URL,
 ) *BlobDataSource {
 	return &BlobDataSource{
 		ctx:                ctx,
 		client:             client,
 		blobServerEndpoint: blobServerEndpoint,
+		socialScanEndpoint: socialScanEndpoint,
 	}
 }
 
+// UnmarshalJSON overwrites to parse data based on different json keys
+func (p *BlobServerResponse) UnmarshalJSON(data []byte) error {
+	var tempMap map[string]interface{}
+	if err := json.Unmarshal(data, &tempMap); err != nil {
+		return err
+	}
+
+	// Parsing data based on different keys
+	if versionedHash, ok := tempMap["versionedHash"]; ok {
+		p.VersionedHash = versionedHash.(string)
+	} else if versionedHash, ok := tempMap["versioned_hash"]; ok {
+		p.VersionedHash = versionedHash.(string)
+	}
+
+	p.Commitment = tempMap["commitment"].(string)
+	p.Data = tempMap["data"].(string)
+
+	return nil
+}
+
 // GetBlobs get blob sidecar by meta
 func (ds *BlobDataSource) GetBlobs(
 	ctx context.Context,
@@ -68,7 +92,7 @@ func (ds *BlobDataSource) GetBlobs(
 	}
 	if err != nil {
 		log.Info("Failed to get blobs from beacon, try to use blob server.", "error", err.Error())
-		if ds.blobServerEndpoint == nil {
+		if ds.blobServerEndpoint == nil && ds.socialScanEndpoint == nil {
 			log.Info("No blob server endpoint set")
 			return nil, err
 		}
@@ -89,8 +113,18 @@ func (ds *BlobDataSource) GetBlobs(
 
 // getBlobFromServer get blob data from server path `/getBlob`.
 func (ds *BlobDataSource) getBlobFromServer(ctx context.Context, blobHash common.Hash) (*BlobDataSeq, error) {
-	route := "/blobs/" + blobHash.String()
-	requestURL, err := url.JoinPath(ds.blobServerEndpoint.String(), route)
+	var (
+		route      string
+		requestURL string
+		err        error
+	)
+	if ds.socialScanEndpoint != nil {
+		route = "/blob/" + blobHash.String()
+		requestURL, err = url.JoinPath(ds.socialScanEndpoint.String(), route)
+	} else {
+		route = "/blobs/" + blobHash.String()
+		requestURL, err = url.JoinPath(ds.blobServerEndpoint.String(), route)
+	}
 	if err != nil {
 		return nil, err
 	}

packages/taiko-client/pkg/rpc/blob_datasource_test.go

@@ -0,0 +1,56 @@
+package rpc
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+
+	"github.com/taikoxyz/taiko-mono/packages/taiko-client/bindings"
+)
+
+func TestGetBlobsFromSocialScan(t *testing.T) {
+	socialScanEndpoint, err := url.Parse("https://api.socialscan.io/blob-archive/v1/holesky-testnet")
+	require.Nil(t, err)
+	require.NotNil(t, socialScanEndpoint)
+	ds := NewBlobDataSource(
+		context.Background(),
+		&Client{},
+		nil,
+		socialScanEndpoint,
+	)
+	sidecars, err := ds.GetBlobs(
+		context.Background(),
+		&bindings.TaikoDataBlockMetadata{
+			BlobHash: common.HexToHash("0x0145185449c57dee4e6c921b702e5d572fbeb026f96c220a6a17b79d157d921b"),
+			BlobUsed: true,
+		},
+	)
+	require.Nil(t, err)
+	require.NotNil(t, sidecars)
+	require.NotNil(t, sidecars[0].Blob)
+}
+
+func TestGetBlobsFromBlobScan(t *testing.T) {
+	blobScanEndpoint, err := url.Parse("https://api.holesky.blobscan.com")
+	require.Nil(t, err)
+	require.NotNil(t, blobScanEndpoint)
+	ds := NewBlobDataSource(
+		context.Background(),
+		&Client{},
+		blobScanEndpoint,
+		nil,
+	)
+	sidecars, err := ds.GetBlobs(
+		context.Background(),
+		&bindings.TaikoDataBlockMetadata{
+			BlobHash: common.HexToHash("0x0145185449c57dee4e6c921b702e5d572fbeb026f96c220a6a17b79d157d921b"),
+			BlobUsed: true,
+		},
+	)
+	require.Nil(t, err)
+	require.NotNil(t, sidecars)
+	require.NotNil(t, sidecars[0].Blob)
+}

packages/taiko-client/proposer/proposer_test.go

@@ -48,6 +48,7 @@ func (s *ProposerTestSuite) SetupTest() {
 		beaconsync.NewSyncProgressTracker(s.RPCClient.L2, 1*time.Hour),
 		0,
 		nil,
+		nil,
 	)
 	s.Nil(err)
 	s.s = syncer

packages/taiko-client/prover/event_handler/transition_proved_test.go

@@ -65,6 +65,7 @@ func (s *EventHandlerTestSuite) SetupTest() {
 		tracker,
 		0,
 		nil,
+		nil,
 	)
 	s.Nil(err)
 

packages/taiko-client/prover/proof_submitter/proof_submitter_test.go

@@ -115,6 +115,7 @@ func (s *ProofSubmitterTestSuite) SetupTest() {
 		tracker,
 		0,
 		nil,
+		nil,
 	)
 	s.Nil(err)