backward compatibility of .lock (#10006)

In PR: 
- new .lock format introduced by
#9766 is not backward
compatible. In the past “empty .lock” did mean “all prohibited” and it
was changed to “all allowed”.
- commit

Not in PR: I have idea to make .lock also forward compatible - by making
it whitelist instead of blacklist: after adding new snap type it will
not be downloaded by accident. Will do it in next PR.

But I need one more confirmation - why do we need exceptions from .lock?
Why we breaking "download once" invariant for some type of files? Can we
avoid it?

erigon-lib/common/dir/rw_dir.go

@@ -72,15 +72,16 @@ func WriteFileWithFsync(name string, data []byte, perm os.FileMode) error {
 		return err
 	}
 	defer f.Close()
-	_, err = f.Write(data)
-	if err != nil {
+	if _, err = f.Write(data); err != nil {
 		return err
 	}
-	err = f.Sync()
-	if err != nil {
+	if err = f.Sync(); err != nil {
+		return err
+	}
+	if err = f.Close(); err != nil {
 		return err
 	}
-	return err
+	return nil
 }
 
 func Recreate(dir string) {

erigon-lib/direct/downloader_client.go

@@ -36,8 +36,8 @@ func (c *DownloaderClient) Add(ctx context.Context, in *proto_downloader.AddRequ
 	return c.server.Add(ctx, in)
 }
 
-func (c *DownloaderClient) ProhibitNewDownloads(ctx context.Context, in *proto_downloader.ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
-	return c.server.ProhibitNewDownloads(ctx, in)
+func (c *DownloaderClient) Prohibit(ctx context.Context, in *proto_downloader.ProhibitRequest, opts ...grpc.CallOption) (*proto_downloader.ProhibitReply, error) {
+	return c.server.Prohibit(ctx, in)
 }
 func (c *DownloaderClient) Delete(ctx context.Context, in *proto_downloader.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
 	return c.server.Delete(ctx, in)

erigon-lib/downloader/downloader_grpc_server.go

@@ -45,8 +45,9 @@ type GrpcServer struct {
 	d *Downloader
 }
 
-func (s *GrpcServer) ProhibitNewDownloads(ctx context.Context, req *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) {
-	return &emptypb.Empty{}, s.d.torrentFS.ProhibitNewDownloads(req.Type)
+func (s *GrpcServer) Prohibit(ctx context.Context, req *proto_downloader.ProhibitRequest) (*proto_downloader.ProhibitReply, error) {
+	whitelist, err := s.d.torrentFS.ProhibitNewDownloads(req.WhitelistAdd, req.WhitelistRemove)
+	return &proto_downloader.ProhibitReply{Whitelist: whitelist}, err
 }
 
 // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)

erigon-lib/downloader/torrent_files.go

@@ -3,7 +3,6 @@ package downloader
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -169,79 +168,90 @@ const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock"
 
 // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)
 // After "download once" - Erigon will produce and seed new files
-// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts)
-func (tf *AtomicTorrentFS) ProhibitNewDownloads(t string) error {
+// After `Prohibit` call - downloader stil will able:
+//   - seed new (generated by Erigon) files
+//   - seed existing on Disk files
+//   - download uncomplete parts of existing on Disk files (if Verify found some bad parts)
+//
+// `Prohibit` has `whitelist` feature - based on file-type
+func (tf *AtomicTorrentFS) ProhibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) {
 	tf.lock.Lock()
 	defer tf.lock.Unlock()
-	return tf.prohibitNewDownloads(t)
+	return tf.prohibitNewDownloads(whitelistAdd, whitelistRemove)
 }
 
-func (tf *AtomicTorrentFS) prohibitNewDownloads(t string) error {
-	// open or create file ProhibitNewDownloadsFileName
-	f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_RDONLY, 0644)
-	if err != nil {
-		return fmt.Errorf("open file: %w", err)
-	}
-	defer f.Close()
-	var prohibitedList []string
-	torrentListJsonBytes, err := io.ReadAll(f)
-	if err != nil {
-		return fmt.Errorf("read file: %w", err)
-	}
-	if len(torrentListJsonBytes) > 0 {
-		if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil {
-			return fmt.Errorf("unmarshal: %w", err)
+func (tf *AtomicTorrentFS) prohibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) {
+	fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName)
+	exist := dir.FileExist(fPath)
+
+	var _currentWhiteList []string
+	if exist {
+		torrentListJsonBytes, err := os.ReadFile(fPath)
+		if err != nil {
+			return nil, fmt.Errorf("read file: %w", err)
+		}
+		if len(torrentListJsonBytes) > 0 {
+			if err := json.Unmarshal(torrentListJsonBytes, &_currentWhiteList); err != nil {
+				return nil, fmt.Errorf("unmarshal: %w", err)
+			}
 		}
 	}
-	if slices.Contains(prohibitedList, t) {
-		return nil
+
+	whiteList := make([]string, 0, len(_currentWhiteList))
+	// copy all item except delted
+	for _, it := range _currentWhiteList {
+		if slices.Contains(whitelistRemove, it) {
+			continue
+		}
+		whiteList = append(whiteList, it)
 	}
-	prohibitedList = append(prohibitedList, t)
-	f.Close()
 
-	// write new prohibited list by opening the file in truncate mode
-	f, err = os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_TRUNC|os.O_WRONLY, 0644)
-	if err != nil {
-		return fmt.Errorf("open file for writing: %w", err)
+	// add all new whitelisted items
+	for _, it := range whitelistAdd {
+		if !slices.Contains(whiteList, it) {
+			whiteList = append(whiteList, it)
+		}
 	}
-	defer f.Close()
-	prohibitedListJsonBytes, err := json.Marshal(prohibitedList)
+	slices.Sort(whiteList)
+
+	whiteListBytes, err := json.Marshal(whiteList)
 	if err != nil {
-		return fmt.Errorf("marshal: %w", err)
+		return _currentWhiteList, fmt.Errorf("marshal: %w", err)
 	}
-	if _, err := f.Write(prohibitedListJsonBytes); err != nil {
-		return fmt.Errorf("write: %w", err)
+	if err := dir.WriteFileWithFsync(fPath, whiteListBytes, 0644); err != nil {
+		return _currentWhiteList, fmt.Errorf("write: %w", err)
 	}
-
-	return f.Sync()
+	return whiteList, nil
 }
 
-func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (bool, error) {
+func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (prohibited bool, err error) {
 	tf.lock.Lock()
 	defer tf.lock.Unlock()
 	return tf.newDownloadsAreProhibited(name)
 }
 
-func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (bool, error) {
-	f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_APPEND|os.O_RDONLY, 0644)
-	if err != nil {
-		return false, err
+func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (prohibited bool, err error) {
+	fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName)
+	exists := dir.FileExist(fPath)
+	if !exists { // no .lock - means all allowed
+		return false, nil
 	}
-	defer f.Close()
-	var prohibitedList []string
-	torrentListJsonBytes, err := io.ReadAll(f)
+
+	var whiteList []string
+	whiteListBytes, err := os.ReadFile(fPath)
 	if err != nil {
 		return false, fmt.Errorf("NewDownloadsAreProhibited: read file: %w", err)
 	}
-	if len(torrentListJsonBytes) > 0 {
-		if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil {
+	if len(whiteListBytes) > 0 {
+		if err := json.Unmarshal(whiteListBytes, &whiteList); err != nil {
 			return false, fmt.Errorf("NewDownloadsAreProhibited: unmarshal: %w", err)
 		}
 	}
-	for _, p := range prohibitedList {
-		if strings.Contains(name, p) {
-			return true, nil
+
+	for _, whiteListedItem := range whiteList {
+		if strings.Contains(name, whiteListedItem) {
+			return false, nil
 		}
 	}
-	return false, nil
+	return true, nil
 }

erigon-lib/downloader/torrent_files_test.go

@@ -0,0 +1,59 @@
+package downloader
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ledgerwatch/erigon-lib/common/datadir"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFSProhibitBackwardCompat(t *testing.T) {
+	require := require.New(t)
+	dirs := datadir.New(t.TempDir())
+
+	//prev version of .lock - is empty .lock file which exitence prohibiting everything
+	t.Run("no prev version .lock", func(t *testing.T) {
+		tf := NewAtomicTorrentFS(dirs.Snap)
+		prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg")
+		require.NoError(err)
+		require.False(prohibited)
+		prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent")
+		require.NoError(err)
+		require.False(prohibited)
+	})
+	t.Run("prev version .lock support", func(t *testing.T) {
+		err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644)
+		require.NoError(err)
+
+		tf := NewAtomicTorrentFS(dirs.Snap)
+		prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg")
+		require.NoError(err)
+		require.True(prohibited)
+		prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent")
+		require.NoError(err)
+		require.True(prohibited)
+	})
+	t.Run("prev version .lock upgrade", func(t *testing.T) {
+		//old lock
+		err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644)
+		require.NoError(err)
+
+		tf := NewAtomicTorrentFS(dirs.Snap)
+		wl, err := tf.prohibitNewDownloads([]string{"transactions"}, nil) //upgrade
+		require.NoError(err)
+		require.Equal([]string{"transactions"}, wl)
+
+		prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg")
+		require.NoError(err)
+		require.True(prohibited)
+		prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent")
+		require.NoError(err)
+		require.True(prohibited)
+
+		prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-transactions.seg")
+		require.NoError(err)
+		require.False(prohibited)
+	})
+}

erigon-lib/go.mod

@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/erigontech/mdbx-go v0.27.24
 	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901
-	github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087
+	github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0
 )

erigon-lib/go.sum

@@ -269,8 +269,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 h1:gAcI47OHnt/1e/APIV0093NVdviIfAnBUzFyybmKL1Q=
 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
-github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 h1:Y59HUAT/+02Qbm6g7MuY7i8E0kUihPe7+ftDnR8oQzQ=
-github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
+github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08 h1:NQRyMIGIapAFnr7hAY0xXQZPMBjtYCUAQ0UF1/saBaE=
+github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
 github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
 github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=