feat: draft the module by implementing a WIP skeleton

lib.go

@@ -0,0 +1,48 @@
+package hibpsync
+
+const (
+	defaultDataDir   = "./.hibp-data"
+	defaultEndpoint  = "https://api.pwnedpasswords.com/range/"
+	defaultCheckETag = true
+)
+
+type syncConfig struct {
+	dataDir   string
+	endpoint  string
+	checkETag bool
+}
+
+type SyncOption func(*syncConfig)
+
+func WithDataDir(dataDir string) SyncOption {
+	return func(c *syncConfig) {
+		c.dataDir = dataDir
+	}
+}
+
+func WithEndpoint(endpoint string) SyncOption {
+	return func(c *syncConfig) {
+		c.endpoint = endpoint
+	}
+}
+
+func WithCheckETag(checkETag bool) SyncOption {
+	return func(c *syncConfig) {
+		c.checkETag = checkETag
+	}
+}
+
+func Sync(options ...SyncOption) {
+	config := &syncConfig{
+		dataDir:   defaultDataDir,
+		endpoint:  defaultEndpoint,
+		checkETag: defaultCheckETag,
+	}
+
+	for _, option := range options {
+		option(config)
+	}
+
+	// TODO: Implement sync
+	// We want to use a pool of workers that draw their range from
+}

ranges.go

@@ -0,0 +1,64 @@
+package hibpsync
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"strconv"
+	"sync"
+)
+
+const writeStateEveryN = 10
+
+type rangeGenerator struct {
+	idx, to       int
+	lock          sync.Mutex
+	stateFilePath string
+}
+
+func newRangeGenerator(from, to int, stateFilePath string) (*rangeGenerator, error) {
+	// Check if the state file exists and read the last state from it.
+	// This is useful to resume the sync process after a crash.
+	if stateFilePath != "" {
+		bytez, err := os.ReadFile(stateFilePath)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return nil, fmt.Errorf("reading state file: %w", err)
+		}
+
+		from, err = strconv.Atoi(string(bytez))
+		if err != nil {
+			return nil, fmt.Errorf("parsing state file: %w", err)
+		}
+	}
+
+	return &rangeGenerator{
+		idx:           from,
+		to:            to,
+		stateFilePath: stateFilePath,
+	}, nil
+}
+
+func (r *rangeGenerator) Next() (int, bool, error) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.idx > r.to {
+		return 0, false, nil
+	}
+
+	current := r.idx
+	r.idx++
+
+	if r.stateFilePath != "" && (current%writeStateEveryN == 0 || current == r.to) {
+		if err := os.WriteFile(r.stateFilePath, []byte(fmt.Sprintf("%d", current)), 0644); err != nil {
+			return 0, false, fmt.Errorf("writing state file: %w", err)
+		}
+	}
+
+	return current, true, nil
+}
+
+func toRangeString(i int) string {
+	return fmt.Sprintf("%05X", i)
+}

storage.go

@@ -0,0 +1,31 @@
+package hibpsync
+
+import "sync"
+
+type fsStorage struct {
+	dataDir   string
+	writeLock sync.Mutex
+}
+
+func (f *fsStorage) Save(key, etag string, data []byte) error {
+	// We need to synchronize calls to Save because we don't want to create the same parent directory for several files
+	// at the same time.
+	f.writeLock.Lock()
+	defer f.writeLock.Unlock()
+
+	// TODO: Implement Save
+
+	return nil
+}
+
+func (f *fsStorage) LoadETag(key string) (string, error) {
+	// TODO: Implement LoadETag
+
+	return "", nil
+}
+
+func (f *fsStorage) LoadData(key string) ([]byte, error) {
+	// TODO: Implement LoadData
+
+	return nil, nil
+}

upstream.go

@@ -0,0 +1,53 @@
+package hibpsync
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+type hibpClient struct {
+	endpoint   string
+	httpClient http.Client
+}
+
+type hibpResponse struct {
+	NotModified bool
+	ETag        string
+	Data        []byte
+}
+
+func (h *hibpClient) RequestRange(rangePrefix, etag string) (*hibpResponse, error) {
+	req, err := http.NewRequest("GET", h.endpoint+rangePrefix, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating request for range %q: %w", rangePrefix, err)
+	}
+
+	if etag != "" {
+		req.Header.Set("If-None-Match", etag)
+	}
+
+	resp, err := h.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("executing request for range %q: %w", rangePrefix, err)
+	}
+
+	if resp.StatusCode == http.StatusNotModified {
+		return &hibpResponse{NotModified: true}, nil
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code requesting range %q: %d", rangePrefix, resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading response body for range %q: %w", rangePrefix, err)
+	}
+
+	return &hibpResponse{
+		ETag: resp.Header.Get("ETag"),
+		Data: body,
+	}, nil
+}