diff --git a/lib.go b/lib.go new file mode 100644 index 0000000..0be1a3c --- /dev/null +++ b/lib.go @@ -0,0 +1,48 @@ +package hibpsync + +const ( + defaultDataDir = "./.hibp-data" + defaultEndpoint = "https://api.pwnedpasswords.com/range/" + defaultCheckETag = true +) + +type syncConfig struct { + dataDir string + endpoint string + checkETag bool +} + +type SyncOption func(*syncConfig) + +func WithDataDir(dataDir string) SyncOption { + return func(c *syncConfig) { + c.dataDir = dataDir + } +} + +func WithEndpoint(endpoint string) SyncOption { + return func(c *syncConfig) { + c.endpoint = endpoint + } +} + +func WithCheckETag(checkETag bool) SyncOption { + return func(c *syncConfig) { + c.checkETag = checkETag + } +} + +func Sync(options ...SyncOption) { + config := &syncConfig{ + dataDir: defaultDataDir, + endpoint: defaultEndpoint, + checkETag: defaultCheckETag, + } + + for _, option := range options { + option(config) + } + + // TODO: Implement sync + // We want to use a pool of workers that draw their range from +} diff --git a/ranges.go b/ranges.go new file mode 100644 index 0000000..5bde52a --- /dev/null +++ b/ranges.go @@ -0,0 +1,64 @@ +package hibpsync + +import ( + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "sync" +) + +const writeStateEveryN = 10 + +type rangeGenerator struct { + idx, to int + lock sync.Mutex + stateFilePath string +} + +func newRangeGenerator(from, to int, stateFilePath string) (*rangeGenerator, error) { + // Check if the state file exists and read the last state from it. + // This is useful to resume the sync process after a crash. + if stateFilePath != "" { + bytez, err := os.ReadFile(stateFilePath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("reading state file: %w", err) + } + + from, err = strconv.Atoi(string(bytez)) + if err != nil { + return nil, fmt.Errorf("parsing state file: %w", err) + } + } + + return &rangeGenerator{ + idx: from, + to: to, + stateFilePath: stateFilePath, + }, nil +} + +func (r *rangeGenerator) Next() (int, bool, error) { + r.lock.Lock() + defer r.lock.Unlock() + + if r.idx > r.to { + return 0, false, nil + } + + current := r.idx + r.idx++ + + if r.stateFilePath != "" && (current%writeStateEveryN == 0 || current == r.to) { + if err := os.WriteFile(r.stateFilePath, []byte(fmt.Sprintf("%d", current)), 0644); err != nil { + return 0, false, fmt.Errorf("writing state file: %w", err) + } + } + + return current, true, nil +} + +func toRangeString(i int) string { + return fmt.Sprintf("%05X", i) +} diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..73731b6 --- /dev/null +++ b/storage.go @@ -0,0 +1,31 @@ +package hibpsync + +import "sync" + +type fsStorage struct { + dataDir string + writeLock sync.Mutex +} + +func (f *fsStorage) Save(key, etag string, data []byte) error { + // We need to synchronize calls to Save because we don't want to create the same parent directory for several files + // at the same time. + f.writeLock.Lock() + defer f.writeLock.Unlock() + + // TODO: Implement Save + + return nil +} + +func (f *fsStorage) LoadETag(key string) (string, error) { + // TODO: Implement LoadETag + + return "", nil +} + +func (f *fsStorage) LoadData(key string) ([]byte, error) { + // TODO: Implement LoadData + + return nil, nil +} diff --git a/upstream.go b/upstream.go new file mode 100644 index 0000000..f0eba51 --- /dev/null +++ b/upstream.go @@ -0,0 +1,53 @@ +package hibpsync + +import ( + "fmt" + "io" + "net/http" +) + +type hibpClient struct { + endpoint string + httpClient http.Client +} + +type hibpResponse struct { + NotModified bool + ETag string + Data []byte +} + +func (h *hibpClient) RequestRange(rangePrefix, etag string) (*hibpResponse, error) { + req, err := http.NewRequest("GET", h.endpoint+rangePrefix, nil) + if err != nil { + return nil, fmt.Errorf("creating request for range %q: %w", rangePrefix, err) + } + + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + + resp, err := h.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("executing request for range %q: %w", rangePrefix, err) + } + + if resp.StatusCode == http.StatusNotModified { + return &hibpResponse{NotModified: true}, nil + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code requesting range %q: %d", rangePrefix, resp.StatusCode) + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response body for range %q: %w", rangePrefix, err) + } + + return &hibpResponse{ + ETag: resp.Header.Get("ETag"), + Data: body, + }, nil +}