Skip to content

Commit

Permalink
fix: document that we use CRLF as line ending, same as upstream API d…
Browse files Browse the repository at this point in the history
…oes, and update tests accordingly
  • Loading branch information
FlorianLoch committed Mar 7, 2024
1 parent 2aae6f6 commit 136faa8
Show file tree
Hide file tree
Showing 6 changed files with 40 additions and 29 deletions.
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ HIBP#.Query("ABCDE") (io.ReadClose, error) // Returns the k-proximity API result
All of them operate on disk but, depending on the medium, should provide access times that are probably good enough for all scenarios.
A memory-based `tmpfs` will speed things up when necessary.

**Attention:**
The [official API](https://haveibeenpwned.com/API/v3#PwnedPasswords) states the following regarding the format:

> Each password is stored as both a SHA-1 and an NTLM hash of a UTF-8 encoded password.
> The downloadable source data delimits the hash and the password count with a colon (:) and each line with a CRLF.
The crucial part being that lines are ended with `\r\n`.
In order to be compatible with the upstream API this library sticks to this...


## CLI

Expand Down
2 changes: 1 addition & 1 deletion cmd/sync/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func run(dataDir string) error {
return nil
}

h := hibp.New(hibp.WithDataDir(dataDir), hibp.WithNoCompression())
h := hibp.New(hibp.WithDataDir(dataDir))

if err := h.Sync(
hibp.SyncWithProgressFn(updateProgressBar),
Expand Down
4 changes: 3 additions & 1 deletion export.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import (
"io"
)

var lineSeparator = []byte("\n")
// The upstream Have-I-Been-Pwned API uses CRLF as line separator - so we are stuck with it,
// although it does not feel right.
var lineSeparator = []byte("\r\n")

func export(from, to int64, store storage, w io.Writer) error {
for i := from; i < to; i++ {
Expand Down
4 changes: 2 additions & 2 deletions export_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ func TestExport(t *testing.T) {
ctrl := gomock.NewController(t)
storageMock := NewMockstorage(ctrl)

storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil)
storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\r\nsuffix:counter12"))), nil)
storageMock.EXPECT().LoadData("00001").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter2"))), nil)
storageMock.EXPECT().LoadData("00002").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter3"))), nil)

Expand All @@ -26,7 +26,7 @@ func TestExport(t *testing.T) {
// HIBP API looks like.
// This has to be the case because `Export` iterates over all ranges; different from `Query` which only
// queries a single range.
if buf.String() != "00000suffix:counter11\n00000suffix:counter12\n00001suffix:counter2\n00002suffix:counter3" {
if buf.String() != "00000suffix:counter11\r\n00000suffix:counter12\r\n00001suffix:counter2\r\n00002suffix:counter3" {
t.Fatalf("unexpected output: %q", buf.String())
}
}
4 changes: 2 additions & 2 deletions lib_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ func TestQuery(t *testing.T) {
ctrl := gomock.NewController(t)
storageMock := NewMockstorage(ctrl)

storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil)
storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\r\nsuffix:counter12"))), nil)

i := HIBP{store: storageMock}

Expand All @@ -29,7 +29,7 @@ func TestQuery(t *testing.T) {

// We expect the lines to not be prefixed with the range as this is what the response from the official
// HIBP API looks like.
if string(lines) != "suffix:counter11\nsuffix:counter12" {
if string(lines) != "suffix:counter11\r\nsuffix:counter12" {
t.Fatalf("unexpected output: %q", string(lines))
}
}
Expand Down
46 changes: 23 additions & 23 deletions sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,63 +23,63 @@ func TestSync(t *testing.T) {
Get("/range/00000").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix1")
BodyString("suffix1:1")
gock.New(baseURL).
Get("/range/00001").
MatchHeader("If-None-Match", "etag received earlier").
Reply(http.StatusNotModified).
AddHeader("ETag", "etag received earlier").
BodyString("suffix2")
BodyString("suffix2:2")
gock.New(baseURL).
Get("/range/00002").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix31:2\nsuffix32:3")
BodyString("suffix31:2\r\nsuffix32:3")
gock.New(baseURL).
Get("/range/00003").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix4")
BodyString("suffix4:4")
gock.New(baseURL).
Get("/range/00004").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix5")
BodyString("suffix5:5")
gock.New(baseURL).
Get("/range/00005").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix6")
BodyString("suffix6:6")
gock.New(baseURL).
Get("/range/00006").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix7")
BodyString("suffix7:7")
gock.New(baseURL).
Get("/range/00007").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix8")
BodyString("suffix8:8")
gock.New(baseURL).
Get("/range/00008").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix9")
BodyString("suffix9:9")
gock.New(baseURL).
Get("/range/00009").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix10")
BodyString("suffix10:10")
gock.New(baseURL).
Get("/range/0000A").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix11")
BodyString("suffix11:11")
gock.New(baseURL).
Get("/range/0000B").
Reply(200).
AddHeader("ETag", "etag").
BodyString("suffix12")
BodyString("suffix12:12")

client := &hibpClient{
endpoint: defaultEndpoint,
Expand All @@ -90,29 +90,29 @@ func TestSync(t *testing.T) {
storageMock := NewMockstorage(ctrl)

storageMock.EXPECT().LoadETag("00000").Return("", nil)
storageMock.EXPECT().Save("00000", "etag", []byte("suffix1")).Return(nil)
storageMock.EXPECT().Save("00000", "etag", []byte("suffix1:1")).Return(nil)
storageMock.EXPECT().LoadETag("00001").Return("etag received earlier", nil)
// 00001 does not need to be written as its ETag has not changed
storageMock.EXPECT().LoadETag("00002").Return("", nil)
storageMock.EXPECT().Save("00002", "etag", []byte("suffix31:2\n00suffix32:3")).Return(nil)
storageMock.EXPECT().Save("00002", "etag", []byte("suffix31:2\r\nsuffix32:3")).Return(nil)
storageMock.EXPECT().LoadETag("00003").Return("", nil)
storageMock.EXPECT().Save("00003", "etag", []byte("suffix4")).Return(nil)
storageMock.EXPECT().Save("00003", "etag", []byte("suffix4:4")).Return(nil)
storageMock.EXPECT().LoadETag("00004").Return("", nil)
storageMock.EXPECT().Save("00004", "etag", []byte("suffix5")).Return(nil)
storageMock.EXPECT().Save("00004", "etag", []byte("suffix5:5")).Return(nil)
storageMock.EXPECT().LoadETag("00005").Return("", nil)
storageMock.EXPECT().Save("00005", "etag", []byte("suffix6")).Return(nil)
storageMock.EXPECT().Save("00005", "etag", []byte("suffix6:6")).Return(nil)
storageMock.EXPECT().LoadETag("00006").Return("", nil)
storageMock.EXPECT().Save("00006", "etag", []byte("suffix7")).Return(nil)
storageMock.EXPECT().Save("00006", "etag", []byte("suffix7:7")).Return(nil)
storageMock.EXPECT().LoadETag("00007").Return("", nil)
storageMock.EXPECT().Save("00007", "etag", []byte("suffix8")).Return(nil)
storageMock.EXPECT().Save("00007", "etag", []byte("suffix8:8")).Return(nil)
storageMock.EXPECT().LoadETag("00008").Return("", nil)
storageMock.EXPECT().Save("00008", "etag", []byte("suffix9")).Return(nil)
storageMock.EXPECT().Save("00008", "etag", []byte("suffix9:9")).Return(nil)
storageMock.EXPECT().LoadETag("00009").Return("", nil)
storageMock.EXPECT().Save("00009", "etag", []byte("suffix10")).Return(nil)
storageMock.EXPECT().Save("00009", "etag", []byte("suffix10:10")).Return(nil)
storageMock.EXPECT().LoadETag("0000A").Return("", nil)
storageMock.EXPECT().Save("0000A", "etag", []byte("suffix11")).Return(nil)
storageMock.EXPECT().Save("0000A", "etag", []byte("suffix11:11")).Return(nil)
storageMock.EXPECT().LoadETag("0000B").Return("", nil)
storageMock.EXPECT().Save("0000B", "etag", []byte("suffix12")).Return(nil)
storageMock.EXPECT().Save("0000B", "etag", []byte("suffix12:12")).Return(nil)

var callCounter atomic.Int64

Expand Down

0 comments on commit 136faa8

Please sign in to comment.