Merge pull request #706 from NaturalSelectionLabs/feat/add_media_farc…

…aster

feat(crawler/farcaster): add media in farcaster

common/worker/snapshot/client_test.go

@@ -3,6 +3,7 @@ package snapshot_test
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/naturalselectionlabs/pregod/common/worker/snapshot"
 	"github.com/naturalselectionlabs/pregod/common/worker/snapshot/graphql"
@@ -61,7 +62,7 @@ func TestGetMultipleVotes(t *testing.T) {
 		OrderDirection: snapshot.OrderDirectionAsc,
 		Where: graphqlx.VoteWhere{
 			// Proposal:   graphql.String("QmPvbwguLfcVryzBRrbY4Pb9bCtxURagdv1XjhtFLf3wHj"),
-			CreatedGte: graphql.Int(1669766184),
+			CreatedGte: graphql.Int(time.Now().Unix()),
 		},
 	}
 

go.mod

@@ -30,8 +30,8 @@ require (
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/lib/pq v1.10.9
 	github.com/mattn/go-mastodon v0.0.6
-	github.com/naturalselectionlabs/kurora v0.59.0
-	github.com/naturalselectionlabs/kurora/client v0.0.0-20230816091615-85b927fdf31d
+	github.com/naturalselectionlabs/kurora v0.60.0
+	github.com/naturalselectionlabs/kurora/client v0.0.0-20230825093104-131c525b47d5
 	github.com/rabbitmq/amqp091-go v1.6.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/samber/lo v1.38.1
@@ -64,6 +64,7 @@ require (
 
 require (
 	github.com/bits-and-blooms/bitset v1.7.0 // indirect
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.10.0 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect

go.sum

@@ -122,6 +122,7 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf
 github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -606,8 +607,12 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h
 github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/naturalselectionlabs/kurora v0.59.0 h1:oaPcV4kmX/o1azizpvzdwb7PYD2QI8zrZnJr9XTTqO0=
 github.com/naturalselectionlabs/kurora v0.59.0/go.mod h1:03nEcXB3BqLqC2F+dREjOYmwuD7OvxmldWdJGvsA4SY=
+github.com/naturalselectionlabs/kurora v0.60.0 h1:zHxKv+DV5ShRLUZMYNR1g80Xg5srtRoWAVvhSmPCPp4=
+github.com/naturalselectionlabs/kurora v0.60.0/go.mod h1:03nEcXB3BqLqC2F+dREjOYmwuD7OvxmldWdJGvsA4SY=
 github.com/naturalselectionlabs/kurora/client v0.0.0-20230816091615-85b927fdf31d h1:K7YK35gPTR+/5eRbN8tJI1hJepm3tyRKA7aQhCtNSPA=
 github.com/naturalselectionlabs/kurora/client v0.0.0-20230816091615-85b927fdf31d/go.mod h1:bgKMhkAEEv3Nm9fgdbgSWDbCfJyWLHr7Me+Np5J71mI=
+github.com/naturalselectionlabs/kurora/client v0.0.0-20230825093104-131c525b47d5 h1:+9y9ZDeFEufC06f1pP1b/kR/a+2DaOZ0xwK51nz1cSk=
+github.com/naturalselectionlabs/kurora/client v0.0.0-20230825093104-131c525b47d5/go.mod h1:bgKMhkAEEv3Nm9fgdbgSWDbCfJyWLHr7Me+Np5J71mI=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=

service/crawler/internal/crawler/farcaster/farcaster.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/gabriel-vasile/mimetype"
 	kurora "github.com/naturalselectionlabs/kurora/client"
 	"github.com/naturalselectionlabs/pregod/common/cache"
 	"github.com/naturalselectionlabs/pregod/common/database"
@@ -22,7 +23,7 @@ import (
 	"github.com/naturalselectionlabs/pregod/service/crawler/internal/config"
 	"github.com/naturalselectionlabs/pregod/service/crawler/internal/crawler"
 	"github.com/samber/lo"
-
+	lop "github.com/samber/lo/parallel"
 	"go.uber.org/zap"
 )
 
@@ -121,6 +122,7 @@ func (s *service) GetKuroraCasts(ctx context.Context) ([]*model.Transaction, str
 			AuthorDisplayname:    rawCast.AuthorDisplayname,
 			AuthorPfpUrl:         rawCast.AuthorPfpUrl,
 			Text:                 rawCast.Text,
+			Media:                rawCast.Media,
 			PublishedAt:          rawCast.PublishedAt,
 			RepliesCount:         rawCast.RepliesCount,
 			ReactionsCount:       rawCast.ReactionsCount,
@@ -185,6 +187,10 @@ func (s *service) buildTransactions(ctx context.Context, cast Cast) ([]*model.Tr
 		TypeOnPlatform: []string{"cast"},
 	}
 
+	if len(cast.Media) > 0 {
+		s.buildPostAttachments(post, cast.Media)
+	}
+
 	if cast.Hash == cast.ThreadHash {
 		// post
 		metadataPost, _ := json.Marshal(post)
@@ -280,6 +286,10 @@ func (s *service) buildTransactions(ctx context.Context, cast Cast) ([]*model.Tr
 			TypeOnPlatform: []string{"cast"},
 		}
 
+		if len(target.Media) > 0 {
+			s.buildPostAttachments(post.Target, target.Media)
+		}
+
 		metadataPost, _ := json.Marshal(post)
 
 		for _, evmAddress := range user.SignerAddresses {
@@ -318,3 +328,36 @@ func (s *service) buildTransactions(ctx context.Context, cast Cast) ([]*model.Tr
 
 	return internalTransactions, nil
 }
+
+func (s *service) buildPostAttachments(post *metadata.Post, embeds []string) {
+	var locker sync.Mutex
+
+	opt := lop.NewOption().WithConcurrency(10)
+
+	lop.ForEach(embeds, func(embed string, i int) {
+		address := embed
+		mimeType, err := detectMimeType(embed)
+		if err != nil {
+			loggerx.Global().Warn("detect mime type error", zap.String("url", address), zap.Error(err))
+			return
+		}
+
+		locker.Lock()
+		post.Media = append(post.Media, metadata.Media{Address: address, MimeType: mimeType})
+		locker.Unlock()
+	}, opt)
+}
+
+func detectMimeType(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", fmt.Errorf("http request error %w", err)
+	}
+	defer resp.Body.Close()
+	mimeType, err := mimetype.DetectReader(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("detect mime type error %w", err)
+	}
+
+	return mimeType.String(), nil
+}

service/crawler/internal/crawler/farcaster/model.go

@@ -15,6 +15,7 @@ type Cast struct {
 	AuthorDisplayname    string
 	AuthorPfpUrl         string
 	Text                 string
+	Media                []string
 	PublishedAt          time.Time
 	RepliesCount         int64
 	ReactionsCount       int64