Merge pull request #16 from titabo2k/master

add hourly interval

Makefile

@@ -10,6 +10,9 @@ $(TARGETS): %: cmd/%/main.go
 	GO111MODULE=$(GO111MODULE) go get ./...
 	GO111MODULE=$(GO111MODULE) CGO_ENABLED=0 go build -o $@ $<
 
+test:
+	CGO_ENABLED=0 go test -v .
+
 clean:
 	rm -f $(TARGETS)
 	rm -f $(PKGNAME)_*deb

cmd/metha-sync/main.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	baseDir                    = flag.String("base-dir", metha.GetBaseDir(), "base dir for harvested files")
+	hourly                     = flag.Bool("hourly", false, "use hourly intervals for harvesting")
 	daily                      = flag.Bool("daily", false, "use daily intervals for harvesting")
 	disableSelectiveHarvesting = flag.Bool("no-intervals", false, "harvest in one go, for funny endpoints")
 	endpointList               = flag.Bool("list", false, "list a selection of OAI endpoints (might be outdated)")
@@ -116,6 +117,7 @@ func main() {
 	harvest.MaxEmptyResponses = *maxEmptyReponses
 	harvest.IgnoreHTTPErrors = *ignoreHTTPErrors
 	harvest.SuppressFormatParameter = *suppressFormatParameter
+	harvest.HourlyInterval = *hourly
 	harvest.DailyInterval = *daily
 	harvest.ExtraHeaders = extra
 

harvest.go

@@ -63,6 +63,7 @@ type Harvest struct {
 	IgnoreHTTPErrors           bool
 	MaxEmptyResponses          int
 	SuppressFormatParameter    bool
+	HourlyInterval             bool
 	DailyInterval              bool
 	ExtraHeaders               http.Header
 
@@ -280,6 +281,8 @@ func (h *Harvest) run() (err error) {
 	var intervals []Interval
 
 	switch {
+	case h.HourlyInterval:
+		intervals = interval.HourlyIntervals()
 	case h.DailyInterval:
 		intervals = interval.DailyIntervals()
 	default:

intervals.go

@@ -55,3 +55,22 @@ func (iv Interval) DailyIntervals() []Interval {
 	}
 	return ivals
 }
+
+// HourlyIntervals segments a given interval into hourly intervals.
+func (iv Interval) HourlyIntervals() []Interval {
+	var ivals []Interval
+	start := iv.Begin
+	for {
+		if start.After(iv.End) {
+			break
+		}
+		end := now.New(start).EndOfHour()
+		if end.After(iv.End) {
+			ivals = append(ivals, Interval{Begin: start, End: end})
+			break
+		}
+		ivals = append(ivals, Interval{Begin: start, End: end})
+		start = now.New(start.Add(time.Hour * 1)).BeginningOfHour()
+	}
+	return ivals
+}

intervals_test.go

@@ -53,3 +53,44 @@ func TestDailyIntervals(t *testing.T) {
 	}
 
 }
+
+func TestHourlyIntervals(t *testing.T) {
+	var cases = []struct {
+		Interval Interval
+		Result   []Interval
+	}{
+		{
+			Interval: Interval{
+				Begin: TimeMustParse("2006-01-02 15:04:05", "2016-01-02 17:00:00"),
+				End:   TimeMustParse("2006-01-02 15:04:05", "2016-01-02 19:00:00"),
+			},
+			Result: []Interval{
+				Interval{
+					TimeMustParse("2006-01-02 15:04:05", "2016-01-02 17:00:00"),
+					TimeMustParse("2006-01-02T15:04:05.999999999", "2016-01-02T17:59:59.999999999"),
+				},
+				Interval{
+					TimeMustParse("2006-01-02 15:04:05", "2016-01-02 18:00:00"),
+					TimeMustParse("2006-01-02T15:04:05.999999999", "2016-01-02T18:59:59.999999999"),
+				},
+				Interval{
+					TimeMustParse("2006-01-02 15:04:05", "2016-01-02 19:00:00"),
+					TimeMustParse("2006-01-02T15:04:05.999999999", "2016-01-02T19:59:59.999999999"),
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		r := c.Interval.HourlyIntervals()
+		if len(r) != len(c.Result) {
+			t.Errorf("got %v, want %v", len(r), len(c.Result))
+		}
+		for i := range r {
+			if r[i].String() != c.Result[i].String() {
+				t.Errorf("got %v, want %s", r[i].String(), c.Result[i].String())
+			}
+		}
+	}
+
+}