diff --git a/Makefile b/Makefile index 3638acf7..f3a605b4 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ $(TARGETS): %: cmd/%/main.go GO111MODULE=$(GO111MODULE) go get ./... GO111MODULE=$(GO111MODULE) CGO_ENABLED=0 go build -o $@ $< +test: + CGO_ENABLED=0 go test -v . + clean: rm -f $(TARGETS) rm -f $(PKGNAME)_*deb diff --git a/cmd/metha-sync/main.go b/cmd/metha-sync/main.go index 49ceb0a6..1ccfa90a 100644 --- a/cmd/metha-sync/main.go +++ b/cmd/metha-sync/main.go @@ -15,6 +15,7 @@ import ( var ( baseDir = flag.String("base-dir", metha.GetBaseDir(), "base dir for harvested files") + hourly = flag.Bool("hourly", false, "use hourly intervals for harvesting") daily = flag.Bool("daily", false, "use daily intervals for harvesting") disableSelectiveHarvesting = flag.Bool("no-intervals", false, "harvest in one go, for funny endpoints") endpointList = flag.Bool("list", false, "list a selection of OAI endpoints (might be outdated)") @@ -116,6 +117,7 @@ func main() { harvest.MaxEmptyResponses = *maxEmptyReponses harvest.IgnoreHTTPErrors = *ignoreHTTPErrors harvest.SuppressFormatParameter = *suppressFormatParameter + harvest.HourlyInterval = *hourly harvest.DailyInterval = *daily harvest.ExtraHeaders = extra diff --git a/harvest.go b/harvest.go index 5ff22362..f90693a6 100644 --- a/harvest.go +++ b/harvest.go @@ -63,6 +63,7 @@ type Harvest struct { IgnoreHTTPErrors bool MaxEmptyResponses int SuppressFormatParameter bool + HourlyInterval bool DailyInterval bool ExtraHeaders http.Header @@ -280,6 +281,8 @@ func (h *Harvest) run() (err error) { var intervals []Interval switch { + case h.HourlyInterval: + intervals = interval.HourlyIntervals() case h.DailyInterval: intervals = interval.DailyIntervals() default: diff --git a/intervals.go b/intervals.go index 466f65fc..2e5765c5 100644 --- a/intervals.go +++ b/intervals.go @@ -55,3 +55,22 @@ func (iv Interval) DailyIntervals() []Interval { } return ivals } + +// HourlyIntervals segments a given interval into hourly intervals. +func (iv Interval) HourlyIntervals() []Interval { + var ivals []Interval + start := iv.Begin + for { + if start.After(iv.End) { + break + } + end := now.New(start).EndOfHour() + if end.After(iv.End) { + ivals = append(ivals, Interval{Begin: start, End: end}) + break + } + ivals = append(ivals, Interval{Begin: start, End: end}) + start = now.New(start.Add(time.Hour * 1)).BeginningOfHour() + } + return ivals +} diff --git a/intervals_test.go b/intervals_test.go index 483013bf..bde46efe 100644 --- a/intervals_test.go +++ b/intervals_test.go @@ -53,3 +53,44 @@ func TestDailyIntervals(t *testing.T) { } } + +func TestHourlyIntervals(t *testing.T) { + var cases = []struct { + Interval Interval + Result []Interval + }{ + { + Interval: Interval{ + Begin: TimeMustParse("2006-01-02 15:04:05", "2016-01-02 17:00:00"), + End: TimeMustParse("2006-01-02 15:04:05", "2016-01-02 19:00:00"), + }, + Result: []Interval{ + Interval{ + TimeMustParse("2006-01-02 15:04:05", "2016-01-02 17:00:00"), + TimeMustParse("2006-01-02T15:04:05.999999999", "2016-01-02T17:59:59.999999999"), + }, + Interval{ + TimeMustParse("2006-01-02 15:04:05", "2016-01-02 18:00:00"), + TimeMustParse("2006-01-02T15:04:05.999999999", "2016-01-02T18:59:59.999999999"), + }, + Interval{ + TimeMustParse("2006-01-02 15:04:05", "2016-01-02 19:00:00"), + TimeMustParse("2006-01-02T15:04:05.999999999", "2016-01-02T19:59:59.999999999"), + }, + }, + }, + } + + for _, c := range cases { + r := c.Interval.HourlyIntervals() + if len(r) != len(c.Result) { + t.Errorf("got %v, want %v", len(r), len(c.Result)) + } + for i := range r { + if r[i].String() != c.Result[i].String() { + t.Errorf("got %v, want %s", r[i].String(), c.Result[i].String()) + } + } + } + +}