Remote/plugin support

.dockerignore

@@ -20,4 +20,5 @@ debug.test
 *.prof
 *.test
 remark42
-/backend/var/
+/backend/var/
+compose-private-backend.yml

.drone.yml

@@ -7,10 +7,6 @@ pipeline:
   build:
     image: golang:1.12-alpine
     commands:
-      - sleep 5
-      - nslookup mongo
-      - nslookup mongo | grep Address | awk '{print $3}' > backend/.mongo
-      - cat backend/.mongo
       - cd backend/app
       - go build -v ./...
 
@@ -94,8 +90,3 @@ pipeline:
     secrets: [ email_username, email_password ]
     when:
       status: [ changed, failure ]
-
-services:
-  mongo:
-    image: mongo:3.6
-    command: [ --smallfiles ]

.gitignore

@@ -19,3 +19,4 @@ debug.test
 remark42
 /bin/
 /backend/var/
+compose-private-backend.yml

.travis.yml

@@ -3,9 +3,6 @@ install:
   - docker-compose --version
 
 script:
-  - docker run -d --name=mongo mongo:3.6 && sleep 3
-  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-  - echo "running mongo on $MONGO_TEST"
   - docker build
       --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
       --build-arg CI=$CI
@@ -19,7 +16,4 @@ script:
       --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
       --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
       --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_TEST
       .
-  - docker rm -f mongo
-

Dockerfile

@@ -19,7 +19,6 @@ ARG DRONE_BRANCH
 ARG DRONE_PULL_REQUEST
 
 ARG SKIP_BACKEND_TEST
-ARG MONGO_TEST
 
 ADD backend /build/backend
 ADD .git /build/.git
@@ -29,18 +28,14 @@ ENV GOFLAGS="-mod=vendor"
 
 # run tests
 RUN \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     cd app && \
     if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-        go test -covermode=count -coverprofile=/profile.cov_tmp ./... && \
+        go test -p 1 -timeout=30s -covermode=count -coverprofile=/profile.cov_tmp ./... && \
         cat /profile.cov_tmp | grep -v "_mock.go" > /profile.cov ; \
     else echo "skip backend test" ; fi
 
-RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
-
 # linters
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
-    if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
         golangci-lint run --out-format=tab --disable-all --tests=false --enable=unconvert \
         --enable=megacheck --enable=structcheck --enable=gas --enable=gocyclo --enable=dupl --enable=misspell \
         --enable=unparam --enable=varcheck --enable=deadcode --enable=typecheck \

backend/app/cmd/avatar.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"path"
-	"time"
 
 	bolt "github.com/coreos/bbolt"
 	log "github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 
 	"github.com/go-pkgz/auth/avatar"
-	"github.com/go-pkgz/mongo"
 )
 
 // AvatarCommand set of flags and command for avatar migration
@@ -18,7 +16,6 @@ import (
 type AvatarCommand struct {
 	AvatarSrc AvatarGroup `group:"src" namespace:"src"`
 	AvatarDst AvatarGroup `group:"dst" namespace:"dst"`
-	Mongo     MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	migrator AvatarMigrator
 	CommonOpts
@@ -78,13 +75,6 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(gr.FS.Path), nil
-	case "mongo":
-		mgServer, err := ac.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, ac.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(gr.Bolt.File)); err != nil {
 			return nil, err
@@ -93,10 +83,3 @@ func (ac *AvatarCommand) makeAvatarStore(gr AvatarGroup) (avatar.Store, error) {
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", gr.Type)
 }
-
-func (ac *AvatarCommand) makeMongo() (result *mongo.Server, err error) {
-	if ac.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(ac.Mongo.URL, 10*time.Second)
-}

backend/app/cmd/avatar_test.go

@@ -13,42 +13,25 @@ import (
 
 func TestAvatar_Execute(t *testing.T) {
 
-	mongoURL := os.Getenv("MONGO_TEST")
-	if mongoURL == "" {
-		mongoURL = "mongodb://localhost:27017/test"
-	}
-	if mongoURL == "skip" {
-		t.Skip("skip mongo app test")
-	}
 	defer os.RemoveAll("/tmp/ava-test")
 
-	// from fs to mongo
+	// from fs to bolt
 	cmd := AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p := flags.NewParser(&cmd, flags.Default)
-	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
-	err = cmd.Execute(nil)
-	assert.NoError(t, err)
-
-	// from fs to bolt
-	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 100}}
-	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
-	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+	_, err := p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
 		"--dst.bolt.file=/tmp/ava-test.db"})
-	require.Nil(t, err)
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.NoError(t, err)
 
 	// failed
 	cmd = AvatarCommand{migrator: &avatarMigratorMock{retCount: 0, retError: errors.New("failed blah")}}
 	cmd.SetCommon(CommonOpts{RemarkURL: "", SharedSecret: "123456"})
 	p = flags.NewParser(&cmd, flags.Default)
-	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=mongo",
-		"--mongo.url=" + mongoURL, "--mongo.db=test_remark"})
-	require.Nil(t, err)
+	_, err = p.ParseArgs([]string{"--src.type=fs", "--src.fs.path=/tmp/ava-test", "--dst.type=bolt",
+		"--dst.bolt.file=/tmp/ava-test2.db"})
+	require.NoError(t, err)
 	err = cmd.Execute(nil)
 	assert.Error(t, err, "failed blah")
 }

backend/app/cmd/server.go

@@ -22,7 +22,6 @@ import (
 	"github.com/go-pkgz/auth/avatar"
 	"github.com/go-pkgz/auth/provider"
 	"github.com/go-pkgz/auth/token"
-	"github.com/go-pkgz/mongo"
 	"github.com/go-pkgz/rest/cache"
 
 	"github.com/umputun/remark/backend/app/migrator"
@@ -33,6 +32,7 @@ import (
 	"github.com/umputun/remark/backend/app/store/admin"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/image"
+	"github.com/umputun/remark/backend/app/store/remote"
 	"github.com/umputun/remark/backend/app/store/service"
 )
 
@@ -41,7 +41,6 @@ type ServerCommand struct {
 	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
 	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
 	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
-	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 	Admin  AdminGroup  `group:"admin" namespace:"admin" env-namespace:"ADMIN"`
 	Notify NotifyGroup `group:"notify" namespace:"notify" env-namespace:"NOTIFY"`
 	Image  ImageGroup  `group:"image" namespace:"image" env-namespace:"IMAGE"`
@@ -89,16 +88,17 @@ type AuthGroup struct {
 
 // StoreGroup defines options group for store params
 type StoreGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"mongo" default:"bolt"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"bolt" choice:"remote" default:"bolt"`
 	Bolt struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
+	Remote RemoteGroup `group:"remote" namespace:"remote" env-namespace:"REMOTE"`
 }
 
 // ImageGroup defines options group for store pictures
 type ImageGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of storage" choice:"fs" choice:"bolt" default:"fs"`
 	FS   struct {
 		Path       string `long:"path" env:"PATH" default:"./var/pictures" description:"images location"`
 		Staging    string `long:"staging" env:"STAGING" default:"./var/pictures.staging" description:"staging location"`
@@ -114,7 +114,7 @@ type ImageGroup struct {
 
 // AvatarGroup defines options group for avatar params
 type AvatarGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" choice:"mongo" default:"fs"`
+	Type string `long:"type" env:"TYPE" description:"type of avatar storage" choice:"fs" choice:"bolt" default:"fs"`
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
@@ -126,27 +126,22 @@ type AvatarGroup struct {
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" choice:"none" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"none" default:"mem"`
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
 		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
 	} `group:"max" namespace:"max" env-namespace:"MAX"`
 }
 
-// MongoGroup holds all mongo params, used by store, avatar and cache
-type MongoGroup struct {
-	URL string `long:"url" env:"URL" description:"mongo url"`
-	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
-}
-
 // AdminGroup defines options group for admin params
 type AdminGroup struct {
-	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"mongo" default:"shared"`
+	Type   string `long:"type" env:"TYPE" description:"type of admin store" choice:"shared" choice:"remote" default:"shared"`
 	Shared struct {
 		Admins []string `long:"id" env:"ID" description:"admin(s) ids" env-delim:","`
 		Email  string   `long:"email" env:"EMAIL" default:"" description:"admin email"`
 	} `group:"shared" namespace:"shared" env-namespace:"SHARED"`
+	Remote RemoteGroup `group:"remote" namespace:"remote" env-namespace:"REMOTE"`
 }
 
 // NotifyGroup defines options for notification
@@ -178,6 +173,14 @@ type StreamGroup struct {
 	MaxActive       int           `long:"max" env:"MAX" default:"500" description:"max number of parallel streams"`
 }
 
+// RemoteGroup defines options for remote modules (plugins)
+type RemoteGroup struct {
+	API          string        `long:"api" env:"API" description:"remote extension api url"`
+	TimeOut      time.Duration `long:"timeout" env:"TIMEOUT" default:"5s" description:"http timeout"`
+	AuthUser     string        `long:"auth_user" env:"AUTH_USER" description:"basic auth user name"`
+	AuthPassword string        `long:"auth_passwd" env:"AUTH_PASSWD" description:"basic auth user password"`
+}
+
 // serverApp holds all active objects
 type serverApp struct {
 	*ServerCommand
@@ -249,7 +252,7 @@ func (s *ServerCommand) newServerApp() (*serverApp, error) {
 	log.Printf("[DEBUG] image service for url=%s, ttl=%v", imageService.ImageAPI, imageService.TTL)
 
 	dataService := &service.DataStore{
-		Interface:              storeEngine,
+		Engine:                 storeEngine,
 		EditDuration:           s.EditDuration,
 		AdminStore:             adminStore,
 		MaxCommentSize:         s.MaxCommentSize,
@@ -414,13 +417,14 @@ func (s *ServerCommand) makeDataStore() (result engine.Interface, err error) {
 			sites = append(sites, engine.BoltSite{SiteID: site, FileName: fmt.Sprintf("%s/%s.db", s.Store.Bolt.Path, site)})
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: s.Store.Bolt.Timeout}, sites...)
-	case "mongo":
-		mgServer, e := s.makeMongo()
-		if e != nil {
-			return result, errors.Wrap(e, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
+	case "remote":
+		r := &engine.Remote{Client: remote.Client{
+			API:        s.Store.Remote.API,
+			Client:     http.Client{Timeout: s.Store.Remote.TimeOut},
+			AuthUser:   s.Store.Remote.AuthUser,
+			AuthPasswd: s.Store.Remote.AuthPassword,
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported store type %s", s.Store.Type)
 	}
@@ -436,13 +440,6 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 			return nil, err
 		}
 		return avatar.NewLocalFS(s.Avatar.FS.Path), nil
-	case "mongo":
-		mgServer, err := s.makeMongo()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "")
-		return avatar.NewGridFS(conn), nil
 	case "bolt":
 		if err := makeDirs(path.Dir(s.Avatar.Bolt.File)); err != nil {
 			return nil, err
@@ -485,13 +482,14 @@ func (s *ServerCommand) makeAdminStore() (admin.Store, error) {
 			}
 		}
 		return admin.NewStaticStore(s.SharedSecret, s.Admin.Shared.Admins, s.Admin.Shared.Email), nil
-	case "mongo":
-		mgServer, e := s.makeMongo()
-		if e != nil {
-			return nil, errors.Wrap(e, "failed to create mongo server")
-		}
-		conn := mongo.NewConnection(mgServer, s.Mongo.DB, "admin")
-		return admin.NewMongoStore(conn, s.SharedSecret), nil
+	case "remote":
+		r := &admin.Remote{Client: remote.Client{
+			API:        s.Admin.Remote.API,
+			Client:     http.Client{Timeout: s.Admin.Remote.TimeOut},
+			AuthUser:   s.Admin.Remote.AuthUser,
+			AuthPasswd: s.Admin.Remote.AuthPassword,
+		}}
+		return r, nil
 	default:
 		return nil, errors.Errorf("unsupported admin store type %s", s.Admin.Type)
 	}
@@ -503,27 +501,12 @@ func (s *ServerCommand) makeCache() (cache.LoadingCache, error) {
 	case "mem":
 		return cache.NewMemoryCache(cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
 			cache.MaxKeys(s.Cache.Max.Items))
-	// case "mongo":
-	// 	mgServer, err := s.makeMongo()
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "failed to create mongo server")
-	// 	}
-	// 	conn := mongo.NewConnection(mgServer, s.Mongo.DB, "cache")
-	// 	return cache.NewMongoCache(conn, cache.MaxCacheSize(s.Cache.Max.Size), cache.MaxValSize(s.Cache.Max.Value),
-	// 		cache.MaxKeys(s.Cache.Max.Items))
 	case "none":
 		return &cache.Nop{}, nil
 	}
 	return nil, errors.Errorf("unsupported cache type %s", s.Cache.Type)
 }
 
-func (s *ServerCommand) makeMongo() (result *mongo.Server, err error) {
-	if s.Mongo.URL == "" {
-		return nil, errors.New("no mongo URL provided")
-	}
-	return mongo.NewServerWithURL(s.Mongo.URL, 10*time.Second)
-}
-
 func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) {
 
 	providers := 0