From f20c58d1dbab9e39738c385b2f249fd32aba5f16 Mon Sep 17 00:00:00 2001 From: huangyuqi Date: Tue, 13 Oct 2015 10:06:05 +0000 Subject: [PATCH 1/4] implement the kafka sink --- Godeps/Godeps.json | 8 + .../src/github.com/golang/snappy/AUTHORS | 14 + .../src/github.com/golang/snappy/CONTRIBUTORS | 36 + .../src/github.com/golang/snappy/LICENSE | 27 + .../src/github.com/golang/snappy/README | 7 + .../src/github.com/golang/snappy/decode.go | 294 ++++ .../src/github.com/golang/snappy/encode.go | 254 +++ .../src/github.com/golang/snappy/snappy.go | 68 + .../src/github.com/optiopay/kafka/.gitignore | 3 + .../src/github.com/optiopay/kafka/.travis.yml | 23 + .../src/github.com/optiopay/kafka/LICENSE | 19 + .../src/github.com/optiopay/kafka/README.md | 96 ++ .../src/github.com/optiopay/kafka/broker.go | 1316 +++++++++++++++ .../github.com/optiopay/kafka/connection.go | 368 ++++ .../optiopay/kafka/distributing_producer.go | 140 ++ .../src/github.com/optiopay/kafka/doc.go | 10 + .../optiopay/kafka/integration/README.md | 14 + .../optiopay/kafka/integration/cluster.go | 226 +++ .../kafka/integration/kafka-docker/Dockerfile | 18 + .../kafka/integration/kafka-docker/LICENSE | 202 +++ .../kafka/integration/kafka-docker/README.md | 57 + .../integration/kafka-docker/broker-list.sh | 5 + .../docker-compose-single-broker.yml | 14 + .../kafka-docker/docker-compose.yml | 15 + .../kafka-docker/download-kafka.sh | 5 + .../kafka-docker/start-kafka-shell.sh | 2 + .../integration/kafka-docker/start-kafka.sh | 47 + .../optiopay/kafka/kafkatest/broker.go | 245 +++ .../optiopay/kafka/kafkatest/doc.go | 8 + .../optiopay/kafka/kafkatest/server.go | 464 ++++++ .../src/github.com/optiopay/kafka/log.go | 22 + .../github.com/optiopay/kafka/multiplexer.go | 122 ++ .../github.com/optiopay/kafka/proto/doc.go | 6 + .../github.com/optiopay/kafka/proto/errors.go | 67 + .../optiopay/kafka/proto/messages.go | 1473 +++++++++++++++++ .../optiopay/kafka/proto/serialization.go | 364 ++++ .../github.com/optiopay/kafka/proto/snappy.go | 50 + docs/sink-configuration.md | 19 + sinks/kafka/driver.go | 159 ++ sinks/kafka/driver_test.go | 237 +++ sinks/modules.go | 1 + 41 files changed, 6525 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/AUTHORS create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/LICENSE create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/README create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/decode.go create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/encode.go create mode 100644 Godeps/_workspace/src/github.com/golang/snappy/snappy.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/.gitignore create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/LICENSE create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/README.md create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/broker.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/connection.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/distributing_producer.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/doc.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/README.md create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/cluster.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/Dockerfile create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/LICENSE create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/README.md create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/broker-list.sh create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose-single-broker.yml create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose.yml create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/download-kafka.sh create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka-shell.sh create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka.sh create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/broker.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/doc.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/server.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/log.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/multiplexer.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/proto/doc.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/proto/errors.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/proto/messages.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/proto/serialization.go create mode 100644 Godeps/_workspace/src/github.com/optiopay/kafka/proto/snappy.go create mode 100644 sinks/kafka/driver.go create mode 100644 sinks/kafka/driver_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 6fbd871887..b7cb56da95 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -389,6 +389,14 @@ { "ImportPath": "speter.net/go/exp/math/dec/inf", "Rev": "42ca6cd68aa922bc3f32f1e056e61b65945d9ad7" + }, + { + "ImportPath": "github.com/golang/snappy", + "Rev": "723cc1e459b8eea2dea4583200fd60757d40097a" + }, + { + "ImportPath": "github.com/optiopay/kafka", + "Rev": "bc8e0950689f19fafd454acd517abe7b251cf54f" } ] } diff --git a/Godeps/_workspace/src/github.com/golang/snappy/AUTHORS b/Godeps/_workspace/src/github.com/golang/snappy/AUTHORS new file mode 100644 index 0000000000..824bf2e148 --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/AUTHORS @@ -0,0 +1,14 @@ +# This is the official list of Snappy-Go authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Damian Gryski +Google Inc. +Jan Mercl <0xjnml@gmail.com> +Sebastien Binet diff --git a/Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS b/Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS new file mode 100644 index 0000000000..9f54f21ff7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS @@ -0,0 +1,36 @@ +# This is the official list of people who can contribute +# (and typically have contributed) code to the Snappy-Go repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name + +# Please keep the list sorted. + +Damian Gryski +Jan Mercl <0xjnml@gmail.com> +Kai Backman +Marc-Antoine Ruel +Nigel Tao +Rob Pike +Russ Cox +Sebastien Binet diff --git a/Godeps/_workspace/src/github.com/golang/snappy/LICENSE b/Godeps/_workspace/src/github.com/golang/snappy/LICENSE new file mode 100644 index 0000000000..6050c10f4c --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/golang/snappy/README b/Godeps/_workspace/src/github.com/golang/snappy/README new file mode 100644 index 0000000000..5074bbab8d --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/README @@ -0,0 +1,7 @@ +The Snappy compression format in the Go programming language. + +To download and install from source: +$ go get github.com/golang/snappy + +Unless otherwise noted, the Snappy-Go source files are distributed +under the BSD-style license found in the LICENSE file. diff --git a/Godeps/_workspace/src/github.com/golang/snappy/decode.go b/Godeps/_workspace/src/github.com/golang/snappy/decode.go new file mode 100644 index 0000000000..e7f1259a34 --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/decode.go @@ -0,0 +1,294 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" + "io" +) + +var ( + // ErrCorrupt reports that the input is invalid. + ErrCorrupt = errors.New("snappy: corrupt input") + // ErrTooLarge reports that the uncompressed length is too large. + ErrTooLarge = errors.New("snappy: decoded block is too large") + // ErrUnsupported reports that the input isn't supported. + ErrUnsupported = errors.New("snappy: unsupported input") +) + +// DecodedLen returns the length of the decoded block. +func DecodedLen(src []byte) (int, error) { + v, _, err := decodedLen(src) + return v, err +} + +// decodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func decodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n <= 0 || v > 0xffffffff { + return 0, 0, ErrCorrupt + } + + const wordSize = 32 << (^uint(0) >> 32 & 1) + if wordSize == 32 && v > 0x7fffffff { + return 0, 0, ErrTooLarge + } + return int(v), n, nil +} + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil dst. +func Decode(dst, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + if len(dst) < dLen { + dst = make([]byte, dLen) + } + + var d, offset, length int + for s < len(src) { + switch src[s] & 0x03 { + case tagLiteral: + x := uint(src[s] >> 2) + switch { + case x < 60: + s++ + case x == 60: + s += 2 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-1]) + case x == 61: + s += 3 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-2]) | uint(src[s-1])<<8 + case x == 62: + s += 4 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16 + case x == 63: + s += 5 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24 + } + length = int(x + 1) + if length <= 0 { + return nil, errors.New("snappy: unsupported literal length") + } + if length > len(dst)-d || length > len(src)-s { + return nil, ErrCorrupt + } + copy(dst[d:], src[s:s+length]) + d += length + s += length + continue + + case tagCopy1: + s += 2 + if s > len(src) { + return nil, ErrCorrupt + } + length = 4 + int(src[s-2])>>2&0x7 + offset = int(src[s-2])&0xe0<<3 | int(src[s-1]) + + case tagCopy2: + s += 3 + if s > len(src) { + return nil, ErrCorrupt + } + length = 1 + int(src[s-3])>>2 + offset = int(src[s-2]) | int(src[s-1])<<8 + + case tagCopy4: + return nil, errors.New("snappy: unsupported COPY_4 tag") + } + + end := d + length + if offset > d || end > len(dst) { + return nil, ErrCorrupt + } + for ; d < end; d++ { + dst[d] = dst[d-offset] + } + } + if d != dLen { + return nil, ErrCorrupt + } + return dst[:d], nil +} + +// NewReader returns a new Reader that decompresses from r, using the framing +// format described at +// https://github.com/google/snappy/blob/master/framing_format.txt +func NewReader(r io.Reader) *Reader { + return &Reader{ + r: r, + decoded: make([]byte, maxUncompressedChunkLen), + buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize), + } +} + +// Reader is an io.Reader than can read Snappy-compressed bytes. +type Reader struct { + r io.Reader + err error + decoded []byte + buf []byte + // decoded[i:j] contains decoded bytes that have not yet been passed on. + i, j int + readHeader bool +} + +// Reset discards any buffered data, resets all state, and switches the Snappy +// reader to read from r. This permits reusing a Reader rather than allocating +// a new one. +func (r *Reader) Reset(reader io.Reader) { + r.r = reader + r.err = nil + r.i = 0 + r.j = 0 + r.readHeader = false +} + +func (r *Reader) readFull(p []byte) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF { + r.err = ErrCorrupt + } + return false + } + return true +} + +// Read satisfies the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, r.err + } + for { + if r.i < r.j { + n := copy(p, r.decoded[r.i:r.j]) + r.i += n + return n, nil + } + if !r.readFull(r.buf[:4]) { + return 0, r.err + } + chunkType := r.buf[0] + if !r.readHeader { + if chunkType != chunkTypeStreamIdentifier { + r.err = ErrCorrupt + return 0, r.err + } + r.readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + r.err = ErrUnsupported + return 0, r.err + } + + // The chunk types are specified at + // https://github.com/google/snappy/blob/master/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[checksumSize:] + + n, err := DecodedLen(buf) + if err != nil { + r.err = err + return 0, r.err + } + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if _, err := Decode(r.decoded, buf); err != nil { + r.err = err + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeUncompressedData: + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:checksumSize] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - checksumSize + if !r.readFull(r.decoded[:n]) { + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeStreamIdentifier: + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(magicBody) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.buf[:len(magicBody)]) { + return 0, r.err + } + for i := 0; i < len(magicBody); i++ { + if r.buf[i] != magicBody[i] { + r.err = ErrCorrupt + return 0, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + r.err = ErrUnsupported + return 0, r.err + } + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen]) { + return 0, r.err + } + } +} diff --git a/Godeps/_workspace/src/github.com/golang/snappy/encode.go b/Godeps/_workspace/src/github.com/golang/snappy/encode.go new file mode 100644 index 0000000000..f3b5484bc7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/encode.go @@ -0,0 +1,254 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "io" +) + +// We limit how far copy back-references can go, the same as the C++ code. +const maxOffset = 1 << 15 + +// emitLiteral writes a literal chunk and returns the number of bytes written. +func emitLiteral(dst, lit []byte) int { + i, n := 0, uint(len(lit)-1) + switch { + case n < 60: + dst[0] = uint8(n)<<2 | tagLiteral + i = 1 + case n < 1<<8: + dst[0] = 60<<2 | tagLiteral + dst[1] = uint8(n) + i = 2 + case n < 1<<16: + dst[0] = 61<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + i = 3 + case n < 1<<24: + dst[0] = 62<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + dst[3] = uint8(n >> 16) + i = 4 + case int64(n) < 1<<32: + dst[0] = 63<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + dst[3] = uint8(n >> 16) + dst[4] = uint8(n >> 24) + i = 5 + default: + panic("snappy: source buffer is too long") + } + if copy(dst[i:], lit) != len(lit) { + panic("snappy: destination buffer is too short") + } + return i + len(lit) +} + +// emitCopy writes a copy chunk and returns the number of bytes written. +func emitCopy(dst []byte, offset, length int) int { + i := 0 + for length > 0 { + x := length - 4 + if 0 <= x && x < 1<<3 && offset < 1<<11 { + dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1 + dst[i+1] = uint8(offset) + i += 2 + break + } + + x = length + if x > 1<<6 { + x = 1 << 6 + } + dst[i+0] = uint8(x-1)<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + i += 3 + length -= x + } + return i +} + +// Encode returns the encoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire encoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil dst. +func Encode(dst, src []byte) []byte { + if n := MaxEncodedLen(len(src)); len(dst) < n { + dst = make([]byte, n) + } + + // The block starts with the varint-encoded length of the decompressed bytes. + d := binary.PutUvarint(dst, uint64(len(src))) + + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + d += emitLiteral(dst[d:], src) + } + return dst[:d] + } + + // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. + const maxTableSize = 1 << 14 + shift, tableSize := uint(32-8), 1<<8 + for tableSize < maxTableSize && tableSize < len(src) { + shift-- + tableSize *= 2 + } + var table [maxTableSize]int + + // Iterate over the source bytes. + var ( + s int // The iterator position. + t int // The last position with the same hash as s. + lit int // The start position of any pending literal bytes. + ) + for s+3 < len(src) { + // Update the hash table. + b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3] + h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24 + p := &table[(h*0x1e35a7bd)>>shift] + // We need to to store values in [-1, inf) in table. To save + // some initialization time, (re)use the table's zero value + // and shift the values against this zero: add 1 on writes, + // subtract 1 on reads. + t, *p = *p-1, s+1 + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] { + s++ + continue + } + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + d += emitLiteral(dst[d:], src[lit:s]) + } + // Extend the match to be as long as possible. + s0 := s + s, t = s+4, t+4 + for s < len(src) && src[s] == src[t] { + s++ + t++ + } + // Emit the copied bytes. + d += emitCopy(dst[d:], s-t, s-s0) + lit = s + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + d += emitLiteral(dst[d:], src[lit:]) + } + return dst[:d] +} + +// MaxEncodedLen returns the maximum length of a snappy block, given its +// uncompressed length. +func MaxEncodedLen(srcLen int) int { + // Compressed data can be defined as: + // compressed := item* literal* + // item := literal* copy + // + // The trailing literal sequence has a space blowup of at most 62/60 + // since a literal of length 60 needs one tag byte + one extra byte + // for length information. + // + // Item blowup is trickier to measure. Suppose the "copy" op copies + // 4 bytes of data. Because of a special check in the encoding code, + // we produce a 4-byte copy only if the offset is < 65536. Therefore + // the copy op takes 3 bytes to encode, and this type of item leads + // to at most the 62/60 blowup for representing literals. + // + // Suppose the "copy" op copies 5 bytes of data. If the offset is big + // enough, it will take 5 bytes to encode the copy op. Therefore the + // worst case here is a one-byte literal followed by a five-byte copy. + // That is, 6 bytes of input turn into 7 bytes of "compressed" data. + // + // This last factor dominates the blowup, so the final estimate is: + return 32 + srcLen + srcLen/6 +} + +// NewWriter returns a new Writer that compresses to w, using the framing +// format described at +// https://github.com/google/snappy/blob/master/framing_format.txt +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)), + } +} + +// Writer is an io.Writer than can write Snappy-compressed bytes. +type Writer struct { + w io.Writer + err error + enc []byte + buf [checksumSize + chunkHeaderSize]byte + wroteHeader bool +} + +// Reset discards the writer's state and switches the Snappy writer to write to +// w. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(writer io.Writer) { + w.w = writer + w.err = nil + w.wroteHeader = false +} + +// Write satisfies the io.Writer interface. +func (w *Writer) Write(p []byte) (n int, errRet error) { + if w.err != nil { + return 0, w.err + } + if !w.wroteHeader { + copy(w.enc, magicChunk) + if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil { + w.err = err + return n, err + } + w.wroteHeader = true + } + for len(p) > 0 { + var uncompressed []byte + if len(p) > maxUncompressedChunkLen { + uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:] + } else { + uncompressed, p = p, nil + } + checksum := crc(uncompressed) + + // Compress the buffer, discarding the result if the improvement + // isn't at least 12.5%. + chunkType := uint8(chunkTypeCompressedData) + chunkBody := Encode(w.enc, uncompressed) + if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 { + chunkType, chunkBody = chunkTypeUncompressedData, uncompressed + } + + chunkLen := 4 + len(chunkBody) + w.buf[0] = chunkType + w.buf[1] = uint8(chunkLen >> 0) + w.buf[2] = uint8(chunkLen >> 8) + w.buf[3] = uint8(chunkLen >> 16) + w.buf[4] = uint8(checksum >> 0) + w.buf[5] = uint8(checksum >> 8) + w.buf[6] = uint8(checksum >> 16) + w.buf[7] = uint8(checksum >> 24) + if _, err := w.w.Write(w.buf[:]); err != nil { + w.err = err + return n, err + } + if _, err := w.w.Write(chunkBody); err != nil { + w.err = err + return n, err + } + n += len(uncompressed) + } + return n, nil +} diff --git a/Godeps/_workspace/src/github.com/golang/snappy/snappy.go b/Godeps/_workspace/src/github.com/golang/snappy/snappy.go new file mode 100644 index 0000000000..e98653acff --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/snappy/snappy.go @@ -0,0 +1,68 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package snappy implements the snappy block-based compression format. +// It aims for very high speeds and reasonable compression. +// +// The C++ snappy implementation is at https://github.com/google/snappy +package snappy + +import ( + "hash/crc32" +) + +/* +Each encoded block begins with the varint-encoded length of the decoded data, +followed by a sequence of chunks. Chunks begin and end on byte boundaries. The +first byte of each chunk is broken into its 2 least and 6 most significant bits +called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. +Zero means a literal tag. All other values mean a copy tag. + +For literal tags: + - If m < 60, the next 1 + m bytes are literal bytes. + - Otherwise, let n be the little-endian unsigned integer denoted by the next + m - 59 bytes. The next 1 + n bytes after that are literal bytes. + +For copy tags, length bytes are copied from offset bytes ago, in the style of +Lempel-Ziv compression algorithms. In particular: + - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). + The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 + of the offset. The next byte is bits 0-7 of the offset. + - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). + The length is 1 + m. The offset is the little-endian unsigned integer + denoted by the next 2 bytes. + - For l == 3, this tag is a legacy format that is no longer supported. +*/ +const ( + tagLiteral = 0x00 + tagCopy1 = 0x01 + tagCopy2 = 0x02 + tagCopy4 = 0x03 +) + +const ( + checksumSize = 4 + chunkHeaderSize = 4 + magicChunk = "\xff\x06\x00\x00" + magicBody + magicBody = "sNaPpY" + // https://github.com/google/snappy/blob/master/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 bytes". + maxUncompressedChunkLen = 65536 +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://github.com/google/snappy/blob/master/framing_format.txt +func crc(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/.gitignore b/Godeps/_workspace/src/github.com/optiopay/kafka/.gitignore new file mode 100644 index 0000000000..5adf2865ba --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/.gitignore @@ -0,0 +1,3 @@ +.* +!.gitignore +!.travis.yml diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/.travis.yml b/Godeps/_workspace/src/github.com/optiopay/kafka/.travis.yml new file mode 100644 index 0000000000..5fc98e7c16 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/.travis.yml @@ -0,0 +1,23 @@ +language: go +go: +- 1.4.2 +- 1.5 +- tip + + +before_install: +- export REPOSITORY_ROOT=${TRAVIS_BUILD_DIR} +- go get golang.org/x/tools/cmd/vet +- go get github.com/kisielk/errcheck + + +script: +- go test -v -race -timeout=90s github.com/optiopay/kafka/... +- go vet github.com/optiopay/kafka/... +- errcheck github.com/optiopay/kafka/... +- go test -bench '.*' -run none github.com/optiopay/kafka/... + +env: +- WITH_INTEGRATION=true GOMAXPROCS=4 + +sudo: false diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/LICENSE b/Godeps/_workspace/src/github.com/optiopay/kafka/LICENSE new file mode 100644 index 0000000000..caa1f3baf9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Optiopay + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/README.md b/Godeps/_workspace/src/github.com/optiopay/kafka/README.md new file mode 100644 index 0000000000..9ce6e7225c --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/README.md @@ -0,0 +1,96 @@ +[![Build Status](https://travis-ci.org/optiopay/kafka.svg?branch=master)](https://travis-ci.org/optiopay/kafka) +[![GoDoc](https://godoc.org/github.com/optiopay/kafka?status.png)](https://godoc.org/github.com/optiopay/kafka) + +# Kafka + +Kafka is Go client library for [Apache Kafka](https://kafka.apache.org/) +server, released under [MIT license](LICENSE]). + +Kafka provides minimal abstraction over wire protocol, support for transparent +failover and easy to use blocking API. + + +* [godoc](https://godoc.org/github.com/optiopay/kafka) generated documentation, +* [code examples](https://godoc.org/github.com/optiopay/kafka#pkg-examples) + +## Example + +Write all messages from stdin to kafka and print all messages from kafka topic +to stdout. + + +```go +package main + +import ( + "bufio" + "log" + "os" + "strings" + + "github.com/optiopay/kafka" + "github.com/optiopay/kafka/proto" +) + +const ( + topic = "my-messages" + partition = 0 +) + +var kafkaAddrs = []string{"localhost:9092", "localhost:9093"} + +// printConsumed read messages from kafka and print them out +func printConsumed(broker kafka.Client) { + conf := kafka.NewConsumerConf(topic, partition) + conf.StartOffset = kafka.StartOffsetNewest + consumer, err := broker.Consumer(conf) + if err != nil { + log.Fatalf("cannot create kafka consumer for %s:%d: %s", topic, partition, err) + } + + for { + msg, err := consumer.Consume() + if err != nil { + if err != kafka.ErrNoData { + log.Printf("cannot consume %q topic message: %s", topic, err) + } + break + } + log.Printf("message %d: %s", msg.Offset, msg.Value) + } + log.Print("consumer quit") +} + +// produceStdin read stdin and send every non empty line as message +func produceStdin(broker kafka.Client) { + producer := broker.Producer(kafka.NewProducerConf()) + input := bufio.NewReader(os.Stdin) + for { + line, err := input.ReadString('\n') + if err != nil { + log.Fatalf("input error: %s", err) + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + + msg := &proto.Message{Value: []byte(line)} + if _, err := producer.Produce(topic, partition, msg); err != nil { + log.Fatalf("cannot produce message to %s:%d: %s", topic, partition, err) + } + } +} + +func main() { + // connect to kafka cluster + broker, err := kafka.Dial(kafkaAddrs, kafka.NewBrokerConf("test-client")) + if err != nil { + log.Fatalf("cannot connect to kafka cluster: %s", err) + } + defer broker.Close() + + go printConsumed(broker) + produceStdin(broker) +} +``` diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/broker.go b/Godeps/_workspace/src/github.com/optiopay/kafka/broker.go new file mode 100644 index 0000000000..c1ea23340d --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/broker.go @@ -0,0 +1,1316 @@ +package kafka + +import ( + "errors" + "fmt" + "io" + "math/rand" + "net" + "sync" + "syscall" + "time" + + "github.com/optiopay/kafka/proto" +) + +const ( + // StartOffsetNewest configures the consumer to fetch messages produced + // after creating the consumer. + StartOffsetNewest = -1 + + // StartOffsetOldest configures the consumer to fetch starting from the + // oldest message available. + StartOffsetOldest = -2 +) + +var ( + // Returned by consumers on Fetch when the retry limit is set and exceeded. + ErrNoData = errors.New("no data") + + // Make sure interfaces are implemented + _ Client = &Broker{} + _ Consumer = &consumer{} + _ Producer = &producer{} + _ OffsetCoordinator = &offsetCoordinator{} +) + +// Client is the interface implemented by Broker. +type Client interface { + Producer(conf ProducerConf) Producer + Consumer(conf ConsumerConf) (Consumer, error) + OffsetCoordinator(conf OffsetCoordinatorConf) (OffsetCoordinator, error) + OffsetEarliest(topic string, partition int32) (offset int64, err error) + OffsetLatest(topic string, partition int32) (offset int64, err error) + Close() +} + +// Consumer is the interface that wraps the Consume method. +// +// Consume reads a message from a consumer, returning an error when +// encountered. +type Consumer interface { + Consume() (*proto.Message, error) +} + +// Producer is the interface that wraps the Produce method. +// +// Produce writes the messages to the given topic and partition. +// It returns the offset of the first message and any error encountered. +// The offset of each message is also updated accordingly. +type Producer interface { + Produce(topic string, partition int32, messages ...*proto.Message) (offset int64, err error) +} + +// OffsetCoordinator is the interface which wraps the Commit and Offset methods. +type OffsetCoordinator interface { + Commit(topic string, partition int32, offset int64) error + Offset(topic string, partition int32) (offset int64, metadata string, err error) +} + +type topicPartition struct { + topic string + partition int32 +} + +func (tp topicPartition) String() string { + return fmt.Sprintf("%s:%d", tp.topic, tp.partition) +} + +type clusterMetadata struct { + created time.Time + nodes map[int32]string // node ID to address + endpoints map[topicPartition]int32 // partition to leader node ID + partitions map[string]int32 // topic to number of partitions +} + +type BrokerConf struct { + // Kafka client ID. + ClientID string + + // LeaderRetryLimit limits the number of connection attempts to a single + // node before failing. Use LeaderRetryWait to control the wait time + // between retries. + // + // Defaults to 10. + LeaderRetryLimit int + + // LeaderRetryWait sets a limit to the waiting time when trying to connect + // to a single node after failure. + // + // Defaults to 500ms. + // + // Timeout on a connection is controlled by the DialTimeout setting. + LeaderRetryWait time.Duration + + // AllowTopicCreation enables a last-ditch "send produce request" which + // happens if we do not know about a topic. This enables topic creation + // if your Kafka cluster is configured to allow it. + // + // Defaults to False. + AllowTopicCreation bool + + // Any new connection dial timeout. + // + // Default is 10 seconds. + DialTimeout time.Duration + + // DialRetryLimit limits the number of connection attempts to every node in + // cluster before failing. Use DialRetryWait to control the wait time + // between retries. + // + // Defaults to 10. + DialRetryLimit int + + // DialRetryWait sets a limit to the waiting time when trying to establish + // broker connection to single node to fetch cluster metadata. + // + // Defaults to 500ms. + DialRetryWait time.Duration + + // DEPRECATED 2015-07-10 - use Logger instead + // + // TODO(husio) remove + // + // Logger used by the broker. + Log interface { + Print(...interface{}) + Printf(string, ...interface{}) + } + + // Logger is general logging interface that can be provided by popular + // logging frameworks. Used to notify and as replacement for stdlib `log` + // package. + Logger Logger +} + +func NewBrokerConf(clientID string) BrokerConf { + return BrokerConf{ + ClientID: clientID, + DialTimeout: 10 * time.Second, + DialRetryLimit: 10, + DialRetryWait: 500 * time.Millisecond, + AllowTopicCreation: false, + LeaderRetryLimit: 10, + LeaderRetryWait: 500 * time.Millisecond, + Logger: &nullLogger{}, + } +} + +// Broker is an abstract connection to kafka cluster, managing connections to +// all kafka nodes. +type Broker struct { + conf BrokerConf + + mu sync.Mutex + metadata clusterMetadata + conns map[int32]*connection +} + +// Dial connects to any node from a given list of kafka addresses and after +// successful metadata fetch, returns broker. +// +// The returned broker is not initially connected to any kafka node. +func Dial(nodeAddresses []string, conf BrokerConf) (*Broker, error) { + broker := &Broker{ + conf: conf, + conns: make(map[int32]*connection), + } + + if len(nodeAddresses) == 0 { + return nil, errors.New("no addresses provided") + } + numAddresses := len(nodeAddresses) + + for i := 0; i < conf.DialRetryLimit; i++ { + if i > 0 { + conf.Logger.Debug("cannot fetch metadata from any connection", + "retry", i, + "sleep", conf.DialRetryWait) + time.Sleep(conf.DialRetryWait) + } + + // This iterates starting at a random location in the slice, to prevent + // hitting the first server repeatedly + offset := rand.Intn(numAddresses) + for idx := 0; idx < numAddresses; idx++ { + addr := nodeAddresses[(idx+offset)%numAddresses] + + conn, err := newTCPConnection(addr, conf.DialTimeout) + if err != nil { + conf.Logger.Debug("cannot connect", + "address", addr, + "err", err) + continue + } + defer func(c *connection) { + _ = c.Close() + }(conn) + resp, err := conn.Metadata(&proto.MetadataReq{ + ClientID: broker.conf.ClientID, + Topics: nil, + }) + if err != nil { + conf.Logger.Debug("cannot fetch metadata", + "address", addr, + "err", err) + continue + } + if len(resp.Brokers) == 0 { + conf.Logger.Debug("response with no broker data", + "address", addr) + continue + } + broker.cacheMetadata(resp) + return broker, nil + } + } + return nil, errors.New("cannot connect") +} + +// Close closes the broker and all active kafka nodes connections. +func (b *Broker) Close() { + b.mu.Lock() + defer b.mu.Unlock() + for nodeID, conn := range b.conns { + if err := conn.Close(); err != nil { + b.conf.Logger.Info("cannot close node connection", + "nodeID", nodeID, + "err", err) + } + } +} + +func (b *Broker) Metadata() (*proto.MetadataResp, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.fetchMetadata() +} + +// refreshMetadata is requesting metadata information from any node and refresh +// internal cached representation. +// Because it's changing internal state, this method requires lock protection, +// but it does not acquire nor release lock itself. +func (b *Broker) refreshMetadata() error { + meta, err := b.fetchMetadata() + if err == nil { + b.cacheMetadata(meta) + } + return err +} + +// muRefreshMetadata calls refreshMetadata, but protects it with broker's lock. +func (b *Broker) muRefreshMetadata() error { + b.mu.Lock() + err := b.refreshMetadata() + b.mu.Unlock() + return err +} + +// fetchMetadata is requesting metadata information from any node and return +// protocol response if successful +// +// If "topics" are specified, only fetch metadata for those topics (can be +// used to create a topic) +// +// Because it's using metadata information to find node connections it's not +// thread safe and using it require locking. +func (b *Broker) fetchMetadata(topics ...string) (*proto.MetadataResp, error) { + checkednodes := make(map[int32]bool) + + // try all existing connections first + for nodeID, conn := range b.conns { + checkednodes[nodeID] = true + resp, err := conn.Metadata(&proto.MetadataReq{ + ClientID: b.conf.ClientID, + Topics: topics, + }) + if err != nil { + b.conf.Logger.Debug("cannot fetch metadata from node", + "nodeID", nodeID, + "err", err) + continue + } + return resp, nil + } + + // try all nodes that we know of that we're not connected to + for nodeID, addr := range b.metadata.nodes { + if _, ok := checkednodes[nodeID]; ok { + continue + } + conn, err := newTCPConnection(addr, b.conf.DialTimeout) + if err != nil { + b.conf.Logger.Debug("cannot connect", + "address", addr, + "err", err) + continue + } + resp, err := conn.Metadata(&proto.MetadataReq{ + ClientID: b.conf.ClientID, + Topics: topics, + }) + + // we had no active connection to this node, so most likely we don't need it + _ = conn.Close() + + if err != nil { + b.conf.Logger.Debug("cannot fetch metadata from node", + "nodeID", nodeID, + "err", err) + continue + } + return resp, nil + } + + return nil, errors.New("cannot fetch metadata. No topics created?") +} + +// cacheMetadata creates new internal metadata representation using data from +// given response. It's call has to be protected with lock. +// +// Do not call with partial metadata response, this assumes we have the full +// set of metadata in the response +func (b *Broker) cacheMetadata(resp *proto.MetadataResp) { + if !b.metadata.created.IsZero() { + b.conf.Logger.Debug("rewriting old metadata", + "age", time.Now().Sub(b.metadata.created)) + } + b.metadata = clusterMetadata{ + created: time.Now(), + nodes: make(map[int32]string), + endpoints: make(map[topicPartition]int32), + partitions: make(map[string]int32), + } + debugmsg := make([]interface{}, 0) + for _, node := range resp.Brokers { + addr := fmt.Sprintf("%s:%d", node.Host, node.Port) + b.metadata.nodes[node.NodeID] = addr + debugmsg = append(debugmsg, node.NodeID, addr) + } + for _, topic := range resp.Topics { + for _, part := range topic.Partitions { + dest := topicPartition{topic.Name, part.ID} + b.metadata.endpoints[dest] = part.Leader + debugmsg = append(debugmsg, dest, part.Leader) + } + b.metadata.partitions[topic.Name] = int32(len(topic.Partitions)) + } + b.conf.Logger.Debug("new metadata cached", debugmsg...) +} + +// PartitionCount returns how many partitions a given topic has. If a topic +// is not known, 0 and an error are returned. +func (b *Broker) PartitionCount(topic string) (int32, error) { + b.mu.Lock() + defer b.mu.Unlock() + + count, ok := b.metadata.partitions[topic] + if ok { + return count, nil + } + + return 0, fmt.Errorf("topic %s not found in metadata", topic) +} + +// muLeaderConnection returns connection to leader for given partition. If +// connection does not exist, broker will try to connect first and add store +// connection for any further use. +// +// Failed connection retry is controlled by broker configuration. +// +// If broker is configured to allow topic creation, then if we don't find +// the leader we will return a random broker. The broker will error if we end +// up producing to it incorrectly (i.e., our metadata happened to be out of +// date). +func (b *Broker) muLeaderConnection(topic string, partition int32) (conn *connection, err error) { + tp := topicPartition{topic, partition} + + b.mu.Lock() + defer b.mu.Unlock() + + for retry := 0; retry < b.conf.LeaderRetryLimit; retry++ { + if retry != 0 { + b.mu.Unlock() + b.conf.Logger.Debug("cannot get leader connection", + "topic", topic, + "partition", partition, + "retry", retry, + "sleep", b.conf.LeaderRetryWait.String()) + time.Sleep(b.conf.LeaderRetryWait) + b.mu.Lock() + } + + nodeID, ok := b.metadata.endpoints[tp] + if !ok { + err = b.refreshMetadata() + if err != nil { + b.conf.Logger.Info("cannot get leader connection: cannot refresh metadata", + "err", err) + continue + } + nodeID, ok = b.metadata.endpoints[tp] + if !ok { + err = proto.ErrUnknownTopicOrPartition + // If we allow topic creation, now is the point where it is likely that this + // is a brand new topic, so try to get metadata on it (which will trigger + // the creation process) + if b.conf.AllowTopicCreation { + _, err := b.fetchMetadata(topic) + if err != nil { + b.conf.Logger.Info("failed to fetch metadata for new topic", + "topic", topic, + "err", err) + } + } else { + b.conf.Logger.Info("cannot get leader connection: unknown topic or partition", + "topic", topic, + "partition", partition, + "endpoint", tp) + } + continue + } + } + + conn, ok = b.conns[nodeID] + if !ok { + addr, ok := b.metadata.nodes[nodeID] + if !ok { + b.conf.Logger.Info("cannot get leader connection: no information about node", + "nodeID", nodeID) + err = proto.ErrBrokerNotAvailable + continue + } + conn, err = newTCPConnection(addr, b.conf.DialTimeout) + if err != nil { + b.conf.Logger.Info("cannot get leader connection: cannot connect to node", + "address", addr, + "err", err) + continue + } + b.conns[nodeID] = conn + } + return conn, nil + } + return nil, err +} + +// coordinatorConnection returns connection to offset coordinator for given group. +// +// Failed connection retry is controlled by broker configuration. +func (b *Broker) muCoordinatorConnection(consumerGroup string) (conn *connection, resErr error) { + b.mu.Lock() + defer b.mu.Unlock() + + for retry := 0; retry < b.conf.LeaderRetryLimit; retry++ { + if retry != 0 { + b.mu.Unlock() + time.Sleep(b.conf.LeaderRetryWait) + b.mu.Lock() + } + + // first try all already existing connections + for _, conn := range b.conns { + resp, err := conn.ConsumerMetadata(&proto.ConsumerMetadataReq{ + ClientID: b.conf.ClientID, + ConsumerGroup: consumerGroup, + }) + if err != nil { + b.conf.Logger.Debug("cannot fetch coordinator metadata", + "consumGrp", consumerGroup, + "err", err) + resErr = err + continue + } + if resp.Err != nil { + b.conf.Logger.Debug("coordinator metadata response error", + "consumGrp", consumerGroup, + "err", resp.Err) + resErr = err + continue + } + + addr := fmt.Sprintf("%s:%d", resp.CoordinatorHost, resp.CoordinatorPort) + conn, err := newTCPConnection(addr, b.conf.DialTimeout) + if err != nil { + b.conf.Logger.Debug("cannot connect to node", + "coordinatorID", resp.CoordinatorID, + "address", addr, + "err", err) + resErr = err + continue + } + b.conns[resp.CoordinatorID] = conn + return conn, nil + } + + // if none of the connections worked out, try with fresh data + if err := b.refreshMetadata(); err != nil { + b.conf.Logger.Debug("cannot refresh metadata", + "err", err) + resErr = err + continue + } + + for nodeID, addr := range b.metadata.nodes { + if _, ok := b.conns[nodeID]; ok { + // connection to node is cached so it was already checked + continue + } + conn, err := newTCPConnection(addr, b.conf.DialTimeout) + if err != nil { + b.conf.Logger.Debug("cannot connect to node", + "nodeID", nodeID, + "address", addr, + "err", err) + resErr = err + continue + } + b.conns[nodeID] = conn + + resp, err := conn.ConsumerMetadata(&proto.ConsumerMetadataReq{ + ClientID: b.conf.ClientID, + ConsumerGroup: consumerGroup, + }) + if err != nil { + b.conf.Logger.Debug("cannot fetch metadata", + "consumGrp", consumerGroup, + "err", err) + resErr = err + continue + } + if resp.Err != nil { + b.conf.Logger.Debug("metadata response error", + "consumGrp", consumerGroup, + "err", resp.Err) + resErr = err + continue + } + + addr := fmt.Sprintf("%s:%d", resp.CoordinatorHost, resp.CoordinatorPort) + conn, err = newTCPConnection(addr, b.conf.DialTimeout) + if err != nil { + b.conf.Logger.Debug("cannot connect to node", + "coordinatorID", resp.CoordinatorID, + "address", addr, + "err", err) + resErr = err + continue + } + b.conns[resp.CoordinatorID] = conn + return conn, nil + } + resErr = proto.ErrNoCoordinator + } + return nil, resErr +} + +// muCloseDeadConnection is closing and removing any reference to given +// connection. Because we remove dead connection, additional request to refresh +// metadata is made +// +// muCloseDeadConnection call it protected with broker's lock. +func (b *Broker) muCloseDeadConnection(conn *connection) { + b.mu.Lock() + defer b.mu.Unlock() + + for nid, c := range b.conns { + if c == conn { + b.conf.Logger.Debug("closing dead connection", + "nodeID", nid) + delete(b.conns, nid) + _ = c.Close() + if err := b.refreshMetadata(); err != nil { + b.conf.Logger.Debug("cannot refresh metadata", + "err", err) + } + return + } + } +} + +// offset will return offset value for given partition. Use timems to specify +// which offset value should be returned. +func (b *Broker) offset(topic string, partition int32, timems int64) (offset int64, err error) { + conn, err := b.muLeaderConnection(topic, partition) + if err != nil { + return 0, err + } + resp, err := conn.Offset(&proto.OffsetReq{ + ClientID: b.conf.ClientID, + ReplicaID: -1, // any client + Topics: []proto.OffsetReqTopic{ + { + Name: topic, + Partitions: []proto.OffsetReqPartition{ + { + ID: partition, + TimeMs: timems, + MaxOffsets: 2, + }, + }, + }, + }, + }) + if err != nil { + if _, ok := err.(*net.OpError); ok || err == io.EOF || err == syscall.EPIPE { + // Connection is broken, so should be closed, but the error is + // still valid and should be returned so that retry mechanism have + // chance to react. + b.conf.Logger.Debug("connection died while sending message", + "topic", topic, + "partition", partition, + "err", err) + b.muCloseDeadConnection(conn) + } + return 0, err + } + found := false + for _, t := range resp.Topics { + if t.Name != topic { + b.conf.Logger.Debug("unexpected topic information", + "expected", topic, + "got", t.Name) + continue + } + for _, part := range t.Partitions { + if part.ID != partition { + b.conf.Logger.Debug("unexpected partition information", + "topic", t.Name, + "expected", partition, + "got", part.ID) + continue + } + found = true + // happens when there are no messages + if len(part.Offsets) == 0 { + offset = 0 + } else { + offset = part.Offsets[0] + } + err = part.Err + } + } + if !found { + return 0, errors.New("incomplete fetch response") + } + return offset, err +} + +// OffsetEarliest returns the oldest offset available on the given partition. +func (b *Broker) OffsetEarliest(topic string, partition int32) (offset int64, err error) { + return b.offset(topic, partition, -2) +} + +// OffsetLatest return the offset of the next message produced in given partition +func (b *Broker) OffsetLatest(topic string, partition int32) (offset int64, err error) { + return b.offset(topic, partition, -1) +} + +type ProducerConf struct { + // Compression method to use, defaulting to proto.CompressionNone. + Compression proto.Compression + + // Timeout of single produce request. By default, 5 seconds. + RequestTimeout time.Duration + + // Message ACK configuration. Use proto.RequiredAcksAll to require all + // servers to write, proto.RequiredAcksLocal to wait only for leader node + // answer or proto.RequiredAcksNone to not wait for any response. + // Setting this to any other, greater than zero value will make producer to + // wait for given number of servers to confirm write before returning. + RequiredAcks int16 + + // RetryLimit specify how many times message producing should be retried in + // case of failure, before returning the error to the caller. By default + // set to 10. + RetryLimit int + + // RetryWait specify wait duration before produce retry after failure. By + // default set to 200ms. + RetryWait time.Duration + + // Logger used by producer. By default, reuse logger assigned to broker. + Logger Logger +} + +// NewProducerConf returns a default producer configuration. +func NewProducerConf() ProducerConf { + return ProducerConf{ + Compression: proto.CompressionNone, + RequestTimeout: 5 * time.Second, + RequiredAcks: proto.RequiredAcksAll, + RetryLimit: 10, + RetryWait: 200 * time.Millisecond, + Logger: nil, + } +} + +// producer is the link to the client with extra configuration. +type producer struct { + conf ProducerConf + broker *Broker +} + +// Producer returns new producer instance, bound to the broker. +func (b *Broker) Producer(conf ProducerConf) Producer { + if conf.Logger == nil { + conf.Logger = b.conf.Logger + } + return &producer{ + conf: conf, + broker: b, + } +} + +// Produce writes messages to the given destination. Writes within the call are +// atomic, meaning either all or none of them are written to kafka. Produce +// has a configurable amount of retries which may be attempted when common +// errors are encountered. This behaviour can be configured with the +// RetryLimit and RetryWait attributes. +// +// Upon a successful call, the message's Offset field is updated. +func (p *producer) Produce(topic string, partition int32, messages ...*proto.Message) (offset int64, err error) { + +retryLoop: + for retry := 0; retry < p.conf.RetryLimit; retry++ { + if retry != 0 { + time.Sleep(p.conf.RetryWait) + } + + offset, err = p.produce(topic, partition, messages...) + + switch err { + case nil: + break retryLoop + case io.EOF, syscall.EPIPE: + // p.produce call is closing connection when this error shows up, + // but it's also returning it so that retry loop can count this + // case + // we cannot handle this error here, because there is no direct + // access to connection + default: + if err := p.broker.muRefreshMetadata(); err != nil { + p.conf.Logger.Debug("cannot refresh metadata", + "err", err) + } + } + p.conf.Logger.Debug("cannot produce messages", + "retry", retry, + "err", err) + } + + if err == nil { + // offset is the offset value of first published messages + for i, msg := range messages { + msg.Offset = int64(i) + offset + } + } + + return offset, err +} + +// produce send produce request to leader for given destination. +func (p *producer) produce(topic string, partition int32, messages ...*proto.Message) (offset int64, err error) { + conn, err := p.broker.muLeaderConnection(topic, partition) + if err != nil { + return 0, err + } + + req := proto.ProduceReq{ + ClientID: p.broker.conf.ClientID, + Compression: p.conf.Compression, + RequiredAcks: p.conf.RequiredAcks, + Timeout: p.conf.RequestTimeout, + Topics: []proto.ProduceReqTopic{ + { + Name: topic, + Partitions: []proto.ProduceReqPartition{ + { + ID: partition, + Messages: messages, + }, + }, + }, + }, + } + + resp, err := conn.Produce(&req) + if err != nil { + if _, ok := err.(*net.OpError); ok || err == io.EOF || err == syscall.EPIPE { + // Connection is broken, so should be closed, but the error is + // still valid and should be returned so that retry mechanism have + // chance to react. + p.conf.Logger.Debug("connection died while sending message", + "topic", topic, + "partition", partition, + "err", err) + p.broker.muCloseDeadConnection(conn) + } + return 0, err + } + + // we expect single partition response + found := false + for _, t := range resp.Topics { + if t.Name != topic { + p.conf.Logger.Debug("unexpected topic information received", + "expected", topic, + "got", t.Name) + continue + } + for _, part := range t.Partitions { + if part.ID != partition { + p.conf.Logger.Debug("unexpected partition information received", + "topic", t.Name, + "expected", partition, + "got", part.ID) + continue + } + found = true + offset = part.Offset + err = part.Err + } + } + + if !found { + return 0, errors.New("incomplete produce response") + } + return offset, err +} + +type ConsumerConf struct { + // Topic name that should be consumed + Topic string + + // Partition ID that should be consumed. + Partition int32 + + // RequestTimeout controls fetch request timeout. This operation is + // blocking the whole connection, so it should always be set to a small + // value. By default it's set to 50ms. + // To control fetch function timeout use RetryLimit and RetryWait. + RequestTimeout time.Duration + + // RetryLimit limits fetching messages a given amount of times before + // returning ErrNoData error. + // + // Default is -1, which turns this limit off. + RetryLimit int + + // RetryWait controls the duration of wait between fetch request calls, + // when no data was returned. + // + // Default is 50ms. + RetryWait time.Duration + + // RetryErrLimit limits the number of retry attempts when an error is + // encountered. + // + // Default is 10. + RetryErrLimit int + + // RetryErrWait controls the wait duration between retries after failed + // fetch request. + // + // Default is 500ms. + RetryErrWait time.Duration + + // MinFetchSize is the minimum size of messages to fetch in bytes. + // + // Default is 1 to fetch any message available. + MinFetchSize int32 + + // MaxFetchSize is the maximum size of data which can be sent by kafka node + // to consumer. + // + // Default is 2000000 bytes. + MaxFetchSize int32 + + // Consumer cursor starting point. Set to StartOffsetNewest to receive only + // newly created messages or StartOffsetOldest to read everything. Assign + // any offset value to manually set cursor -- consuming starts with the + // message whose offset is equal to given value (including first message). + // + // Default is StartOffsetOldest. + StartOffset int64 + + // Logger used by consumer. By default, reuse logger assigned to broker. + Logger Logger +} + +// NewConsumerConf returns the default consumer configuration. +func NewConsumerConf(topic string, partition int32) ConsumerConf { + return ConsumerConf{ + Topic: topic, + Partition: partition, + RequestTimeout: time.Millisecond * 50, + RetryLimit: -1, + RetryWait: time.Millisecond * 50, + RetryErrLimit: 10, + RetryErrWait: time.Millisecond * 500, + MinFetchSize: 1, + MaxFetchSize: 2000000, + StartOffset: StartOffsetOldest, + Logger: nil, + } +} + +// Consumer represents a single partition reading buffer. Consumer is also +// providing limited failure handling and message filtering. +type consumer struct { + broker *Broker + conf ConsumerConf + + mu sync.Mutex + offset int64 // offset of next NOT consumed message + conn *connection + msgbuf []*proto.Message +} + +// Consumer creates a new consumer instance, bound to the broker. +func (b *Broker) Consumer(conf ConsumerConf) (Consumer, error) { + conn, err := b.muLeaderConnection(conf.Topic, conf.Partition) + if err != nil { + return nil, err + } + if conf.Logger == nil { + conf.Logger = b.conf.Logger + } + offset := conf.StartOffset + if offset < 0 { + switch offset { + case StartOffsetNewest: + off, err := b.OffsetLatest(conf.Topic, conf.Partition) + if err != nil { + return nil, err + } + offset = off + case StartOffsetOldest: + off, err := b.OffsetEarliest(conf.Topic, conf.Partition) + if err != nil { + return nil, err + } + offset = off + default: + return nil, fmt.Errorf("invalid start offset: %d", conf.StartOffset) + } + } + c := &consumer{ + broker: b, + conn: conn, + conf: conf, + msgbuf: make([]*proto.Message, 0), + offset: offset, + } + return c, nil +} + +// Consume is returning single message from consumed partition. Consumer can +// retry fetching messages even if responses return no new data. Retry +// behaviour can be configured through RetryLimit and RetryWait consumer +// parameters. +// +// Consume can retry sending request on common errors. This behaviour can be +// configured with RetryErrLimit and RetryErrWait consumer configuration +// attributes. +func (c *consumer) Consume() (*proto.Message, error) { + c.mu.Lock() + defer c.mu.Unlock() + + var retry int + for len(c.msgbuf) == 0 { + var err error + c.msgbuf, err = c.fetch() + if err != nil { + return nil, err + } + if len(c.msgbuf) == 0 { + if c.conf.RetryWait > 0 { + time.Sleep(c.conf.RetryWait) + } + retry += 1 + if c.conf.RetryLimit != -1 && retry > c.conf.RetryLimit { + return nil, ErrNoData + } + } + } + + msg := c.msgbuf[0] + c.msgbuf = c.msgbuf[1:] + c.offset = msg.Offset + 1 + return msg, nil +} + +// fetch and return next batch of messages. In case of certain set of errors, +// retry sending fetch request. Retry behaviour can be configured with +// RetryErrLimit and RetryErrWait consumer configuration attributes. +func (c *consumer) fetch() ([]*proto.Message, error) { + req := proto.FetchReq{ + ClientID: c.broker.conf.ClientID, + MaxWaitTime: c.conf.RequestTimeout, + MinBytes: c.conf.MinFetchSize, + Topics: []proto.FetchReqTopic{ + { + Name: c.conf.Topic, + Partitions: []proto.FetchReqPartition{ + { + ID: c.conf.Partition, + FetchOffset: c.offset, + MaxBytes: c.conf.MaxFetchSize, + }, + }, + }, + }, + } + + var resErr error +consumeRetryLoop: + for retry := 0; retry < c.conf.RetryErrLimit; retry++ { + if retry != 0 { + time.Sleep(c.conf.RetryErrWait) + } + + if c.conn == nil { + conn, err := c.broker.muLeaderConnection(c.conf.Topic, c.conf.Partition) + if err != nil { + resErr = err + continue + } + c.conn = conn + } + + resp, err := c.conn.Fetch(&req) + resErr = err + + if _, ok := err.(*net.OpError); ok || err == io.EOF || err == syscall.EPIPE { + c.conf.Logger.Debug("connection died while fetching message", + "topic", c.conf.Topic, + "partition", c.conf.Partition, + "err", err) + c.broker.muCloseDeadConnection(c.conn) + c.conn = nil + continue + } + + if err != nil { + c.conf.Logger.Debug("cannot fetch messages: unknown error", + "retry", retry, + "err", err) + c.broker.muCloseDeadConnection(c.conn) + c.conn = nil + continue + } + + for _, topic := range resp.Topics { + if topic.Name != c.conf.Topic { + c.conf.Logger.Warn("unexpected topic information received", + "got", topic.Name, + "expected", c.conf.Topic) + continue + } + for _, part := range topic.Partitions { + if part.ID != c.conf.Partition { + c.conf.Logger.Warn("unexpected partition information received", + "topic", topic.Name, + "expected", c.conf.Partition, + "got", part.ID) + continue + } + switch part.Err { + case proto.ErrLeaderNotAvailable, proto.ErrNotLeaderForPartition, proto.ErrBrokerNotAvailable: + c.conf.Logger.Debug("cannot fetch messages", + "retry", retry, + "err", part.Err) + if err := c.broker.muRefreshMetadata(); err != nil { + c.conf.Logger.Debug("cannot refresh metadata", + "err", err) + } + // The connection is fine, so don't close it, + // but we may very well need to talk to a different broker now. + // Set the conn to nil so that next time around the loop + // we'll check the metadata again to see who we're supposed to talk to. + c.conn = nil + continue consumeRetryLoop + } + return part.Messages, part.Err + } + } + return nil, errors.New("incomplete fetch response") + } + + return nil, resErr +} + +type OffsetCoordinatorConf struct { + ConsumerGroup string + + // RetryErrLimit limits messages fetch retry upon failure. By default 10. + RetryErrLimit int + + // RetryErrWait controls wait duration between retries after failed fetch + // request. By default 500ms. + RetryErrWait time.Duration + + // Logger used by consumer. By default, reuse logger assigned to broker. + Logger Logger +} + +// NewOffsetCoordinatorConf returns default OffsetCoordinator configuration. +func NewOffsetCoordinatorConf(consumerGroup string) OffsetCoordinatorConf { + return OffsetCoordinatorConf{ + ConsumerGroup: consumerGroup, + RetryErrLimit: 10, + RetryErrWait: time.Millisecond * 500, + Logger: nil, + } +} + +type offsetCoordinator struct { + conf OffsetCoordinatorConf + broker *Broker + + mu sync.Mutex + conn *connection +} + +// OffsetCoordinator returns offset management coordinator for single consumer +// group, bound to broker. +func (b *Broker) OffsetCoordinator(conf OffsetCoordinatorConf) (OffsetCoordinator, error) { + conn, err := b.muCoordinatorConnection(conf.ConsumerGroup) + if err != nil { + return nil, err + } + if conf.Logger == nil { + conf.Logger = b.conf.Logger + } + c := &offsetCoordinator{ + broker: b, + conf: conf, + conn: conn, + } + return c, nil +} + +// Commit is saving offset information for given topic and partition. +// +// Commit can retry saving offset information on common errors. This behaviour +// can be configured with with RetryErrLimit and RetryErrWait coordinator +// configuration attributes. +func (c *offsetCoordinator) Commit(topic string, partition int32, offset int64) error { + return c.commit(topic, partition, offset, "") +} + +// Commit works exactly like Commit method, but store extra metadata string +// together with offset information. +func (c *offsetCoordinator) CommitFull(topic string, partition int32, offset int64, metadata string) error { + return c.commit(topic, partition, offset, metadata) +} + +// commit is saving offset and metadata information. Provides limited error +// handling configurable through OffsetCoordinatorConf. +func (c *offsetCoordinator) commit(topic string, partition int32, offset int64, metadata string) (resErr error) { + c.mu.Lock() + defer c.mu.Unlock() + + for retry := 0; retry < c.conf.RetryErrLimit; retry++ { + if retry != 0 { + c.mu.Unlock() + time.Sleep(c.conf.RetryErrWait) + c.mu.Lock() + } + + // connection can be set to nil if previously reference connection died + if c.conn == nil { + conn, err := c.broker.muCoordinatorConnection(c.conf.ConsumerGroup) + if err != nil { + resErr = err + c.conf.Logger.Debug("cannot connect to coordinator", + "consumGrp", c.conf.ConsumerGroup, + "err", err) + continue + } + c.conn = conn + } + + resp, err := c.conn.OffsetCommit(&proto.OffsetCommitReq{ + ClientID: c.broker.conf.ClientID, + ConsumerGroup: c.conf.ConsumerGroup, + Topics: []proto.OffsetCommitReqTopic{ + { + Name: topic, + Partitions: []proto.OffsetCommitReqPartition{ + {ID: partition, Offset: offset, TimeStamp: time.Now(), Metadata: metadata}, + }, + }, + }, + }) + resErr = err + + if _, ok := err.(*net.OpError); ok || err == io.EOF || err == syscall.EPIPE { + c.conf.Logger.Debug("connection died while commiting", + "topic", topic, + "partition", partition, + "consumGrp", c.conf.ConsumerGroup) + c.broker.muCloseDeadConnection(c.conn) + c.conn = nil + } else if err == nil { + for _, t := range resp.Topics { + if t.Name != topic { + c.conf.Logger.Debug("unexpected topic information received", + "got", t.Name, + "expected", topic) + continue + + } + for _, part := range t.Partitions { + if part.ID != partition { + c.conf.Logger.Debug("unexpected partition information received", + "topic", topic, + "got", part.ID, + "expected", partition) + continue + } + return part.Err + } + } + return errors.New("response does not contain commit information") + } + } + return resErr +} + +// Offset is returning last offset and metadata information committed for given +// topic and partition. +// Offset can retry sending request on common errors. This behaviour can be +// configured with with RetryErrLimit and RetryErrWait coordinator +// configuration attributes. +func (c *offsetCoordinator) Offset(topic string, partition int32) (offset int64, metadata string, resErr error) { + c.mu.Lock() + defer c.mu.Unlock() + + for retry := 0; retry < c.conf.RetryErrLimit; retry++ { + if retry != 0 { + c.mu.Unlock() + time.Sleep(c.conf.RetryErrWait) + c.mu.Lock() + } + + // connection can be set to nil if previously reference connection died + if c.conn == nil { + conn, err := c.broker.muCoordinatorConnection(c.conf.ConsumerGroup) + if err != nil { + c.conf.Logger.Debug("cannot connect to coordinator", + "consumGrp", c.conf.ConsumerGroup, + "err", err) + resErr = err + continue + } + c.conn = conn + } + resp, err := c.conn.OffsetFetch(&proto.OffsetFetchReq{ + ConsumerGroup: c.conf.ConsumerGroup, + Topics: []proto.OffsetFetchReqTopic{ + { + Name: topic, + Partitions: []int32{partition}, + }, + }, + }) + resErr = err + + switch err { + case io.EOF, syscall.EPIPE: + c.conf.Logger.Debug("connection died while fetching offset", + "topic", topic, + "partition", partition, + "consumGrp", c.conf.ConsumerGroup) + c.broker.muCloseDeadConnection(c.conn) + c.conn = nil + case nil: + for _, t := range resp.Topics { + if t.Name != topic { + c.conf.Logger.Debug("unexpected topic information received", + "got", t.Name, + "expected", topic) + continue + } + for _, part := range t.Partitions { + if part.ID != partition { + c.conf.Logger.Debug("unexpected partition information received", + "topic", topic, + "expected", partition, + "get", part.ID) + continue + } + if part.Err != nil { + return 0, "", part.Err + } + return part.Offset, part.Metadata, nil + } + } + return 0, "", errors.New("response does not contain offset information") + } + } + + return 0, "", resErr +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/connection.go b/Godeps/_workspace/src/github.com/optiopay/kafka/connection.go new file mode 100644 index 0000000000..a7e6bd8ea0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/connection.go @@ -0,0 +1,368 @@ +package kafka + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "math" + "net" + "sync" + "time" + + "github.com/optiopay/kafka/proto" +) + +// ErrClosed is returned as result of any request made using closed connection. +var ErrClosed = errors.New("closed") + +// Low level abstraction over connection to Kafka. +type connection struct { + rw io.ReadWriteCloser + stop chan struct{} + nextID chan int32 + logger Logger + + mu sync.Mutex + respc map[int32]chan []byte + stopErr error +} + +// newConnection returns new, initialized connection or error +func newTCPConnection(address string, timeout time.Duration) (*connection, error) { + conn, err := net.DialTimeout("tcp", address, timeout) + if err != nil { + return nil, err + } + c := &connection{ + stop: make(chan struct{}), + nextID: make(chan int32), + rw: conn, + respc: make(map[int32]chan []byte), + logger: &nullLogger{}, + } + go c.nextIDLoop() + go c.readRespLoop() + return c, nil +} + +// nextIDLoop generates correlation IDs, making sure they are always in order +// and within the scope of request-response mapping array. +func (c *connection) nextIDLoop() { + var id int32 = 1 + for { + select { + case <-c.stop: + close(c.nextID) + return + case c.nextID <- id: + id++ + if id == math.MaxInt32 { + id = 1 + } + } + } +} + +// readRespLoop constantly reading response messages from the socket and after +// partial parsing, sends byte representation of the whole message to request +// sending process. +func (c *connection) readRespLoop() { + defer func() { + c.mu.Lock() + for _, cc := range c.respc { + close(cc) + } + c.respc = make(map[int32]chan []byte) + c.mu.Unlock() + }() + + rd := bufio.NewReader(c.rw) + for { + correlationID, b, err := proto.ReadResp(rd) + if err != nil { + c.mu.Lock() + if c.stopErr == nil { + c.stopErr = err + close(c.stop) + } + c.mu.Unlock() + return + } + + c.mu.Lock() + rc, ok := c.respc[correlationID] + delete(c.respc, correlationID) + c.mu.Unlock() + if !ok { + c.logger.Warn( + "msg", "response to unknown request", + "correlationID", correlationID) + continue + } + + select { + case <-c.stop: + c.mu.Lock() + if c.stopErr == nil { + c.stopErr = ErrClosed + } + c.mu.Unlock() + case rc <- b: + } + close(rc) + } +} + +// respWaiter register listener to response message with given correlationID +// and return channel that single response message will be pushed to once it +// will arrive. +// After pushing response message, channel is closed. +// +// Upon connection close, all unconsumed channels are closed. +func (c *connection) respWaiter(correlationID int32) (respc chan []byte, err error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.stopErr != nil { + return nil, c.stopErr + } + if _, ok := c.respc[correlationID]; ok { + c.logger.Error("msg", "correlation conflict", "correlationID", correlationID) + return nil, fmt.Errorf("correlation conflict: %d", correlationID) + } + respc = make(chan []byte) + c.respc[correlationID] = respc + return respc, nil +} + +// releaseWaiter removes response channel from waiters pool and close it. +// Calling this method for unknown correlationID has no effect. +func (c *connection) releaseWaiter(correlationID int32) { + c.mu.Lock() + rc, ok := c.respc[correlationID] + if ok { + delete(c.respc, correlationID) + close(rc) + } + c.mu.Unlock() +} + +// Close close underlying transport connection and cancel all pending response +// waiters. +func (c *connection) Close() error { + c.mu.Lock() + if c.stopErr == nil { + c.stopErr = ErrClosed + close(c.stop) + } + c.mu.Unlock() + return c.rw.Close() +} + +// Metadata sends given metadata request to kafka node and returns related +// metadata response. +// Calling this method on closed connection will always return ErrClosed. +func (c *connection) Metadata(req *proto.MetadataReq) (*proto.MetadataResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + return proto.ReadMetadataResp(bytes.NewReader(b)) +} + +// Produce sends given produce request to kafka node and returns related +// response. Sending request with no ACKs flag will result with returning nil +// right after sending request, without waiting for response. +// Calling this method on closed connection will always return ErrClosed. +func (c *connection) Produce(req *proto.ProduceReq) (*proto.ProduceResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + + if req.RequiredAcks == proto.RequiredAcksNone { + _, err := req.WriteTo(c.rw) + return nil, err + } + + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + return proto.ReadProduceResp(bytes.NewReader(b)) +} + +// Fetch sends given fetch request to kafka node and returns related response. +// Calling this method on closed connection will always return ErrClosed. +func (c *connection) Fetch(req *proto.FetchReq) (*proto.FetchResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + resp, err := proto.ReadFetchResp(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + // Compressed messages are returned in full batches for efficiency + // (the broker doesn't need to decompress). + // This means that it's possible to get some leading messages + // with a smaller offset than requested. Trim those. + for ti := range resp.Topics { + topic := &resp.Topics[ti] + reqTopic := &req.Topics[ti] + for pi := range topic.Partitions { + partition := &topic.Partitions[pi] + reqPartition := &reqTopic.Partitions[pi] + i := 0 + for _, msg := range partition.Messages { + if msg.Offset >= reqPartition.FetchOffset { + break + } + i++ + } + partition.Messages = partition.Messages[i:] + } + } + return resp, nil +} + +// Offset sends given offset request to kafka node and returns related response. +// Calling this method on closed connection will always return ErrClosed. +func (c *connection) Offset(req *proto.OffsetReq) (*proto.OffsetResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + + // TODO(husio) documentation is not mentioning this directly, but I assume + // -1 is for non node clients + req.ReplicaID = -1 + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + return proto.ReadOffsetResp(bytes.NewReader(b)) +} + +func (c *connection) ConsumerMetadata(req *proto.ConsumerMetadataReq) (*proto.ConsumerMetadataResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + return proto.ReadConsumerMetadataResp(bytes.NewReader(b)) +} + +func (c *connection) OffsetCommit(req *proto.OffsetCommitReq) (*proto.OffsetCommitResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + return proto.ReadOffsetCommitResp(bytes.NewReader(b)) +} + +func (c *connection) OffsetFetch(req *proto.OffsetFetchReq) (*proto.OffsetFetchResp, error) { + var ok bool + if req.CorrelationID, ok = <-c.nextID; !ok { + return nil, c.stopErr + } + respc, err := c.respWaiter(req.CorrelationID) + if err != nil { + c.logger.Error("msg", "failed waiting for response", "err", err) + return nil, fmt.Errorf("wait for response: %s", err) + } + if _, err := req.WriteTo(c.rw); err != nil { + c.logger.Error("msg", "cannot write", "err", err) + c.releaseWaiter(req.CorrelationID) + return nil, err + } + b, ok := <-respc + if !ok { + return nil, c.stopErr + } + return proto.ReadOffsetFetchResp(bytes.NewReader(b)) +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/distributing_producer.go b/Godeps/_workspace/src/github.com/optiopay/kafka/distributing_producer.go new file mode 100644 index 0000000000..7b40e42853 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/distributing_producer.go @@ -0,0 +1,140 @@ +package kafka + +import ( + "errors" + "fmt" + "hash/fnv" + "math/rand" + "sync" + "time" + + "github.com/optiopay/kafka/proto" +) + +// DistributingProducer is the interface similar to Producer, but never require +// to explicitly specify partition. +// +// Distribute writes messages to the given topic, automatically choosing +// partition, returning the post-commit offset and any error encountered. The +// offset of each message is also updated accordingly. +type DistributingProducer interface { + Distribute(topic string, messages ...*proto.Message) (offset int64, err error) +} + +type randomProducer struct { + rand *rand.Rand + producer Producer + partitions int32 +} + +// NewRandomProducer wraps given producer and return DistributingProducer that +// publish messages to kafka, randomly picking partition number from range +// [0, numPartitions) +func NewRandomProducer(p Producer, numPartitions int32) DistributingProducer { + return &randomProducer{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + producer: p, + partitions: numPartitions, + } +} + +// Distribute write messages to given kafka topic, randomly destination choosing +// partition. All messages written within single Produce call are atomically +// written to the same destination. +func (p *randomProducer) Distribute(topic string, messages ...*proto.Message) (offset int64, err error) { + part := p.rand.Intn(int(p.partitions)) + return p.producer.Produce(topic, int32(part), messages...) +} + +type roundRobinProducer struct { + producer Producer + partitions int32 + mu sync.Mutex + next int32 +} + +// NewRoundRobinProducer wraps given producer and return DistributingProducer +// that publish messages to kafka, choosing destination partition from cycle +// build from [0, numPartitions) range. +func NewRoundRobinProducer(p Producer, numPartitions int32) DistributingProducer { + return &roundRobinProducer{ + producer: p, + partitions: numPartitions, + next: 0, + } +} + +// Distribute write messages to given kafka topic, choosing next destination +// partition from internal cycle. All messages written within single Produce +// call are atomically written to the same destination. +func (p *roundRobinProducer) Distribute(topic string, messages ...*proto.Message) (offset int64, err error) { + p.mu.Lock() + part := p.next + p.next++ + if p.next >= p.partitions { + p.next = 0 + } + p.mu.Unlock() + + return p.producer.Produce(topic, int32(part), messages...) +} + +type hashProducer struct { + producer Producer + partitions int32 +} + +// NewHashProducer wraps given producer and return DistributingProducer that +// publish messages to kafka, computing partition number from message key hash, +// using fnv hash and [0, numPartitions) range. +func NewHashProducer(p Producer, numPartitions int32) DistributingProducer { + return &hashProducer{ + producer: p, + partitions: numPartitions, + } +} + +// Distribute write messages to given kafka topic, computing partition number from +// the message key value. Message key must be not nil and all messages written +// within single Produce call are atomically written to the same destination. +// +// All messages passed within single Produce call must hash to the same +// destination, otherwise no message is written and error is returned. +func (p *hashProducer) Distribute(topic string, messages ...*proto.Message) (offset int64, err error) { + if len(messages) == 0 { + return 0, errors.New("no messages") + } + part, err := p.messagePartition(messages[0]) + if err != nil { + return 0, fmt.Errorf("cannot hash message: %s", err) + } + // make sure that all messages within single call are to the same destination + for i := 2; i < len(messages); i++ { + mp, err := p.messagePartition(messages[i]) + if err != nil { + return 0, fmt.Errorf("cannot hash message: %s", err) + } + if part != mp { + return 0, errors.New("cannot publish messages to different destinations") + } + } + + return p.producer.Produce(topic, part, messages...) +} + +// messagePartition compute message's key hash and return corresponding +// partition number. +func (p *hashProducer) messagePartition(m *proto.Message) (int32, error) { + if m.Key == nil { + return 0, errors.New("no key") + } + hasher := fnv.New32a() + if _, err := hasher.Write(m.Key); err != nil { + return 0, fmt.Errorf("cannot hash key: %s", err) + } + sum := int32(hasher.Sum32()) + if sum < 0 { + sum = -sum + } + return sum / p.partitions, nil +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/doc.go b/Godeps/_workspace/src/github.com/optiopay/kafka/doc.go new file mode 100644 index 0000000000..ff8e4601e1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/doc.go @@ -0,0 +1,10 @@ +/* + +Package kafka a provides high level client API for Apache Kafka. + +Use 'Broker' for node connection management, 'Producer' for sending messages, +and 'Consumer' for fetching. All those structures implement Client, Consumer +and Producer interface, that is also implemented in kafkatest package. + +*/ +package kafka diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/README.md b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/README.md new file mode 100644 index 0000000000..6a161ffc90 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/README.md @@ -0,0 +1,14 @@ +# integration + +Integration with kafka & zoopeeker test helpers. + +`KafkaCluster` depends on `docker` and `docker-compose` commands. + +**IMPORTANT**: Make sure to update `KAFKA_ADVERTISED_HOST_NAME` in +`kafka-docker/docker-compose.yml` before running tests. + +## kafka-docker + +[kafka-docker](/integration/kafka-docker) directory is copy of +[wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) +repository. diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/cluster.go b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/cluster.go new file mode 100644 index 0000000000..eaaad9d1a6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/cluster.go @@ -0,0 +1,226 @@ +package integration + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "testing" +) + +type KafkaCluster struct { + // cluster size == number of kafka nodes + size int + kafkaDockerDir string + verbose bool + + mu sync.Mutex + containers []*Container +} + +type Container struct { + cluster *KafkaCluster + + ID string `json:"Id"` + Image string + Args []string + Config struct { + Cmd []string + Env []string + ExposedPorts map[string]interface{} + } + NetworkSettings struct { + Gateway string + IPAddress string + Ports map[string][]PortMapping + } +} + +type PortMapping struct { + HostIP string `json:"HostIp"` + HostPort string +} + +func NewKafkaCluster(kafkaDockerDir string, size int) *KafkaCluster { + if size < 4 { + fmt.Fprintln(os.Stderr, + "WARNING: creating cluster smaller than 4 nodes is not sufficient for all topics") + } + return &KafkaCluster{ + kafkaDockerDir: kafkaDockerDir, + size: size, + verbose: testing.Verbose(), + } +} + +// RunningKafka returns true if container is running kafka node +func (c *Container) RunningKafka() bool { + return c.Args[1] == "start-kafka.sh" +} + +// Start starts current container +func (c *Container) Start() error { + return c.cluster.ContainerStart(c.ID) +} + +// Stop stops current container +func (c *Container) Stop() error { + return c.cluster.ContainerStop(c.ID) +} + +func (c *Container) Kill() error { + return c.cluster.ContainerKill(c.ID) +} + +// Start start zookeeper and kafka nodes using docker-compose command. Upon +// successful process spawn, cluster is scaled to required amount of nodes. +func (cluster *KafkaCluster) Start() error { + cluster.mu.Lock() + defer cluster.mu.Unlock() + + // ensure cluster is not running + if err := cluster.Stop(); err != nil { + return fmt.Errorf("cannot ensure stop cluster: %s", err) + } + if err := cluster.removeStoppedContainers(); err != nil { + return fmt.Errorf("cannot cleanup dead containers: %s", err) + } + + upCmd := cluster.cmd("docker-compose", "up", "-d") + if err := upCmd.Run(); err != nil { + return fmt.Errorf("docker-compose error: %s", err) + } + + scaleCmd := cluster.cmd("docker-compose", "scale", fmt.Sprintf("kafka=%d", cluster.size)) + if err := scaleCmd.Run(); err != nil { + _ = cluster.Stop() + return fmt.Errorf("cannot scale kafka: %s", err) + } + + containers, err := cluster.Containers() + if err != nil { + _ = cluster.Stop() + return fmt.Errorf("cannot get containers info: %s", err) + } + cluster.containers = containers + return nil +} + +// Containers inspect all containers running within cluster and return +// information about them. +func (cluster *KafkaCluster) Containers() ([]*Container, error) { + psCmd := cluster.cmd("docker-compose", "ps", "-q") + var buf bytes.Buffer + psCmd.Stdout = &buf + if err := psCmd.Run(); err != nil { + return nil, fmt.Errorf("cannot list processes: %s", err) + } + + rd := bufio.NewReader(&buf) + inspectArgs := []string{"inspect"} + for { + line, err := rd.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("cannot read \"ps\" output: %s", err) + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + inspectArgs = append(inspectArgs, line) + } + + inspectCmd := cluster.cmd("docker", inspectArgs...) + buf.Reset() + inspectCmd.Stdout = &buf + if err := inspectCmd.Run(); err != nil { + return nil, fmt.Errorf("inspect failed: %s", err) + } + var containers []*Container + if err := json.NewDecoder(&buf).Decode(&containers); err != nil { + return nil, fmt.Errorf("cannot decode inspection: %s", err) + } + for _, c := range containers { + c.cluster = cluster + } + return containers, nil +} + +// Stop stop all services running for the cluster by sending SIGINT to +// docker-compose process. +func (cluster *KafkaCluster) Stop() error { + cmd := cluster.cmd("docker-compose", "stop", "-t", "0") + if err := cmd.Run(); err != nil { + return fmt.Errorf("docker-compose stop failed: %s", err) + } + _ = cluster.removeStoppedContainers() + return nil +} + +// KafkaAddrs return list of kafka node addresses as strings, in form +// : +func (cluster *KafkaCluster) KafkaAddrs() ([]string, error) { + containers, err := cluster.Containers() + if err != nil { + return nil, fmt.Errorf("cannot get containers info: %s", err) + } + addrs := make([]string, 0) + for _, container := range containers { + ports, ok := container.NetworkSettings.Ports["9092/tcp"] + if !ok || len(ports) == 0 { + continue + } + addrs = append(addrs, fmt.Sprintf("%s:%s", ports[0].HostIP, ports[0].HostPort)) + } + return addrs, nil +} + +func (cluster *KafkaCluster) ContainerStop(containerID string) error { + stopCmd := cluster.cmd("docker", "stop", containerID) + if err := stopCmd.Run(); err != nil { + return fmt.Errorf("cannot stop %q container: %s", containerID, err) + } + return nil +} + +func (cluster *KafkaCluster) ContainerKill(containerID string) error { + killCmd := cluster.cmd("docker", "kill", containerID) + if err := killCmd.Run(); err != nil { + return fmt.Errorf("cannot kill %q container: %s", containerID, err) + } + return nil +} + +func (cluster *KafkaCluster) ContainerStart(containerID string) error { + startCmd := cluster.cmd("docker", "start", containerID) + if err := startCmd.Run(); err != nil { + return fmt.Errorf("cannot start %q container: %s", containerID, err) + } + return nil +} + +func (cluster *KafkaCluster) cmd(name string, args ...string) *exec.Cmd { + c := exec.Command(name, args...) + if cluster.verbose { + c.Stderr = os.Stderr + c.Stdout = os.Stdout + } + c.Dir = cluster.kafkaDockerDir + return c +} + +func (cluster *KafkaCluster) removeStoppedContainers() error { + rmCmd := cluster.cmd("docker-compose", "rm", "-f") + if err := rmCmd.Run(); err != nil { + return fmt.Errorf("docker-compose rm error: %s", err) + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/Dockerfile b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/Dockerfile new file mode 100644 index 0000000000..e15ac6954f --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:trusty + +MAINTAINER Wurstmeister + +ENV KAFKA_VERSION="0.8.2.1" SCALA_VERSION="2.10" + +RUN apt-get update && apt-get install -y unzip openjdk-6-jdk wget curl git docker.io jq + +ADD download-kafka.sh /tmp/download-kafka.sh +RUN /tmp/download-kafka.sh +RUN tar xf /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt + +VOLUME ["/kafka"] + +ENV KAFKA_HOME /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} +ADD start-kafka.sh /usr/bin/start-kafka.sh +ADD broker-list.sh /usr/bin/broker-list.sh +CMD start-kafka.sh diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/LICENSE b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/LICENSE new file mode 100644 index 0000000000..e06d208186 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/README.md b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/README.md new file mode 100644 index 0000000000..e3e89b6add --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/README.md @@ -0,0 +1,57 @@ +kafka-docker +============ + +Dockerfile for [Apache Kafka](http://kafka.apache.org/) + +The image is available directly from https://registry.hub.docker.com/ + +##Pre-Requisites + +- install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/) +- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in ```docker-compose.yml``` to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) +- if you want to customise any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'``` + +##Usage + +Start a cluster: + +- ```docker-compose up -d ``` + +Add more brokers: + +- ```docker-compose scale kafka=3``` + +Destroy a cluster: + +- ```docker-compose stop``` + +##Note + +The default ```docker-compose.yml``` should be seen as a starting point. By default each broker will get a new port number and broker id on restart. Depending on your use case this might not be desirable. If you need to use specific ports and broker ids, modify the docker-compose configuration accordingly, e.g. [docker-compose-single-broker.yml](https://github.com/wurstmeister/kafka-docker/blob/master/docker-compose-single-broker.yml): + +- ```docker-compose -f docker-compose-single-broker.yml up``` + +##Broker IDs + +If you don't specify a broker id in your docker-compose file, it will automatically be generated based on the name that docker-compose gives the container. This allows scaling up and down. In this case it is recommended to use the ```--no-recreate``` option of docker-compose to ensure that containers are not re-created and thus keep their names and ids. + + +##Automatically create topics + +If you want to have kafka-docker automatically create topics in Kafka during +creation, a ```KAFKA_CREATE_TOPICS``` environment variable can be +added in ```docker-compose.yml```. + +Here is an example snippet from ```docker-compose.yml```: + + environment: + KAFKA_CREATE_TOPICS: "Topic1:1:3,Topic2:1:1" + +```Topic 1``` will have 1 partition and 3 replicas, ```Topic 2``` will have 1 partition and 1 replica. + +##Tutorial + +[http://wurstmeister.github.io/kafka-docker/](http://wurstmeister.github.io/kafka-docker/) + + + diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/broker-list.sh b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/broker-list.sh new file mode 100644 index 0000000000..7f046393ba --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/broker-list.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') +BROKERS=$(for CONTAINER in $CONTAINERS; do docker port $CONTAINER 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) +echo $BROKERS | sed -e 's/ /,/g' diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose-single-broker.yml b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose-single-broker.yml new file mode 100644 index 0000000000..cd69c89e5b --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose-single-broker.yml @@ -0,0 +1,14 @@ +zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" +kafka: + image: wurstmeister/kafka:0.8.2.0 + ports: + - "9092:9092" + links: + - zookeeper:zk + environment: + KAFKA_ADVERTISED_HOST_NAME: 192.168.59.103 + volumes: + - /var/run/docker.sock:/var/run/docker.sock diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose.yml b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose.yml new file mode 100644 index 0000000000..e53f5ce123 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/docker-compose.yml @@ -0,0 +1,15 @@ +zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181" +kafka: + build: . + ports: + - "9092" + links: + - zookeeper:zk + environment: + KAFKA_CREATE_TOPICS: "Topic3:1:3,Topic4:1:4" + KAFKA_ADVERTISED_HOST_NAME: 172.17.42.1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/download-kafka.sh b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/download-kafka.sh new file mode 100644 index 0000000000..e3aa4c2ad1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/download-kafka.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | sed -rn 's/.*"preferred":.*"(.*)"/\1/p') +url="${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" +wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka-shell.sh b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka-shell.sh new file mode 100644 index 0000000000..bcb8a477ef --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka-shell.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka:0.8.2.0 /bin/bash diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka.sh b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka.sh new file mode 100644 index 0000000000..824aceceb4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/integration/kafka-docker/start-kafka.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +if [[ -z "$KAFKA_ADVERTISED_PORT" ]]; then + export KAFKA_ADVERTISED_PORT=$(docker port `hostname` 9092 | sed -r "s/.*:(.*)/\1/g") +fi +if [[ -z "$KAFKA_BROKER_ID" ]]; then + export KAFKA_BROKER_ID=$(docker inspect `hostname` | jq --raw-output '.[0] | .Name' | awk -F_ '{print $3}') +fi +if [[ -z "$KAFKA_LOG_DIRS" ]]; then + export KAFKA_LOG_DIRS="/kafka/kafka-logs-$KAFKA_BROKER_ID" +fi +if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then + export KAFKA_ZOOKEEPER_CONNECT=$(env | grep ZK.*PORT_2181_TCP= | sed -e 's|.*tcp://||' | paste -sd ,) +fi + +if [[ -n "$KAFKA_HEAP_OPTS" ]]; then + sed -r -i "s/(export KAFKA_HEAP_OPTS)=\"(.*)\"/\1=\"$KAFKA_HEAP_OPTS\"/g" $KAFKA_HOME/bin/kafka-server-start.sh + unset KAFKA_HEAP_OPTS +fi + +for VAR in `env` +do + if [[ $VAR =~ ^KAFKA_ && ! $VAR =~ ^KAFKA_HOME ]]; then + kafka_name=`echo "$VAR" | sed -r "s/KAFKA_(.*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .` + env_var=`echo "$VAR" | sed -r "s/(.*)=.*/\1/g"` + if egrep -q "(^|^#)$kafka_name=" $KAFKA_HOME/config/server.properties; then + sed -r -i "s@(^|^#)($kafka_name)=(.*)@\2=${!env_var}@g" $KAFKA_HOME/config/server.properties #note that no config values may contain an '@' char + else + echo "$kafka_name=${!env_var}" >> $KAFKA_HOME/config/server.properties + fi + fi +done + + +$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties & +KAFKA_SERVER_PID=$! + +while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do sleep 1; done + +if [[ -n $KAFKA_CREATE_TOPICS ]]; then + IFS=','; for topicToCreate in $KAFKA_CREATE_TOPICS; do + IFS=':' read -a topicConfig <<< "$topicToCreate" + $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $KAFKA_ZOOKEEPER_CONNECT --replication-factor ${topicConfig[2]} --partition ${topicConfig[1]} --topic "${topicConfig[0]}" + done +fi + +wait $KAFKA_SERVER_PID diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/broker.go b/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/broker.go new file mode 100644 index 0000000000..9419841295 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/broker.go @@ -0,0 +1,245 @@ +package kafkatest + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/optiopay/kafka" + "github.com/optiopay/kafka/proto" +) + +var ( + ErrTimeout = errors.New("timeout") + + ErrNotImplemented = errors.New("not implemented") + + // test implementation should implement the interface + _ kafka.Client = &Broker{} + _ kafka.Producer = &Producer{} + _ kafka.Consumer = &Consumer{} +) + +// Broker is mock version of kafka's broker. It's implementing Broker interface +// and provides easy way of mocking server actions. +type Broker struct { + produced chan *ProducedMessages + + mu sync.Mutex + consumers map[string]map[int32]*Consumer + + // OffsetEarliestHandler is callback function called whenever + // OffsetEarliest method of the broker is called. Overwrite to change + // default behaviour -- always returning ErrUnknownTopicOrPartition + OffsetEarliestHandler func(string, int32) (int64, error) + + // OffsetLatestHandler is callback function called whenever OffsetLatest + // method of the broker is called. Overwrite to change default behaviour -- + // always returning ErrUnknownTopicOrPartition + OffsetLatestHandler func(string, int32) (int64, error) +} + +func NewBroker() *Broker { + return &Broker{ + consumers: make(map[string]map[int32]*Consumer), + produced: make(chan *ProducedMessages), + } +} + +// Close is no operation method, required by Broker interface. +func (b *Broker) Close() { +} + +// OffsetEarliest return result of OffsetEarliestHandler callback set on the +// broker. If not set, always return ErrUnknownTopicOrPartition +func (b *Broker) OffsetEarliest(topic string, partition int32) (int64, error) { + if b.OffsetEarliestHandler != nil { + return b.OffsetEarliestHandler(topic, partition) + } + return 0, proto.ErrUnknownTopicOrPartition +} + +// OffsetLatest return result of OffsetLatestHandler callback set on the +// broker. If not set, always return ErrUnknownTopicOrPartition +func (b *Broker) OffsetLatest(topic string, partition int32) (int64, error) { + if b.OffsetLatestHandler != nil { + return b.OffsetLatestHandler(topic, partition) + } + return 0, proto.ErrUnknownTopicOrPartition +} + +// Consumer returns consumer mock and never error. +// +// At most one consumer for every topic-partition pair can be created -- +// calling this for the same topic-partition will always return the same +// consumer instance. +func (b *Broker) Consumer(conf kafka.ConsumerConf) (kafka.Consumer, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if t, ok := b.consumers[conf.Topic]; ok { + if c, ok := t[conf.Partition]; ok { + return c, nil + } + } else { + b.consumers[conf.Topic] = make(map[int32]*Consumer) + } + + c := &Consumer{ + conf: conf, + Broker: b, + Messages: make(chan *proto.Message), + Errors: make(chan error), + } + b.consumers[conf.Topic][conf.Partition] = c + return c, nil +} + +// Producer returns producer mock instance. +func (b *Broker) Producer(kafka.ProducerConf) kafka.Producer { + return &Producer{ + Broker: b, + ResponseOffset: 1, + } +} + +// OffsetCoordinator returns offset coordinator mock instance. It's always +// successful, so you can always ignore returned error. +func (b *Broker) OffsetCoordinator(conf kafka.OffsetCoordinatorConf) (kafka.OffsetCoordinator, error) { + c := &OffsetCoordinator{ + Broker: b, + conf: conf, + } + return c, nil +} + +// ReadProducers return ProduceMessages representing produce call of one of +// created by broker producers or ErrTimeout. +func (b *Broker) ReadProducers(timeout time.Duration) (*ProducedMessages, error) { + select { + case p := <-b.produced: + return p, nil + case <-time.After(timeout): + return nil, ErrTimeout + } +} + +// Consumer mocks kafka's consumer. Use Messages and Errors channels to mock +// Consume method results. +type Consumer struct { + conf kafka.ConsumerConf + + Broker *Broker + + // Messages is channel consumed by fetch method call. Pushing message into + // this channel will result in Consume method call returning message data. + Messages chan *proto.Message + + // Errors is channel consumed by fetch method call. Pushing error into this + // channel will result in Consume method call returning error. + Errors chan error +} + +// Consume returns message or error pushed through consumers Messages and Errors +// channel. Function call will block until data on at least one of those +// channels is available. +func (c *Consumer) Consume() (*proto.Message, error) { + select { + case msg := <-c.Messages: + msg.Topic = c.conf.Topic + msg.Partition = c.conf.Partition + return msg, nil + case err := <-c.Errors: + return nil, err + } +} + +// Producer mocks kafka's producer. +type Producer struct { + Broker *Broker + + // ResponseOffset is offset counter returned and incremented by every + // Produce method call. By default set to 1. + ResponseOffset int64 + + // ResponseError if set, force Produce method call to instantly return + // error, without publishing messages. By default nil. + ResponseError error +} + +// ProducedMessages represents all arguments used for single Produce method +// call. +type ProducedMessages struct { + Topic string + Partition int32 + Messages []*proto.Message +} + +// Produce is settings messages Crc and Offset attributes and pushing all +// passed arguments to broker. Produce call is blocking until pushed message +// will be read with broker's ReadProduces. +func (p *Producer) Produce(topic string, partition int32, messages ...*proto.Message) (int64, error) { + if p.ResponseError != nil { + return 0, p.ResponseError + } + off := p.ResponseOffset + + for i, msg := range messages { + msg.Offset = off + int64(i) + msg.Crc = proto.ComputeCrc(msg, proto.CompressionNone) + } + + p.Broker.produced <- &ProducedMessages{ + Topic: topic, + Partition: partition, + Messages: messages, + } + p.ResponseOffset += int64(len(messages)) + return off, nil +} + +type OffsetCoordinator struct { + conf kafka.OffsetCoordinatorConf + Broker *Broker + + // Offsets is used to store all offset commits when using mocked + // coordinator's default behaviour. + Offsets map[string]int64 + + // CommitHandler is callback function called whenever Commit method of the + // OffsetCoordinator is called. If CommitHandler is nil, Commit method will + // return data using Offset attribute as store. + CommitHandler func(consumerGroup string, topic string, partition int32, offset int64) error + + // OffsetHandler is callback function called whenever Offset method of the + // OffsetCoordinator is called. If OffsetHandler is nil, Commit method will + // use Offset attribute to retrieve the offset. + OffsetHandler func(consumerGroup string, topic string, partition int32) (offset int64, metadata string, err error) +} + +// Commit return result of CommitHandler callback set on coordinator. If +// handler is nil, this method will use Offsets attribute to store data for +// further use. +func (c *OffsetCoordinator) Commit(topic string, partition int32, offset int64) error { + if c.CommitHandler != nil { + return c.CommitHandler(c.conf.ConsumerGroup, topic, partition, offset) + } + c.Offsets[fmt.Sprintf("%s:%d", topic, partition)] = offset + return nil +} + +// Offset return result of OffsetHandler callback set on coordinator. If +// handler is nil, this method will use Offsets attribute to retrieve committed +// offset. If no offset for given topic and partition pair was saved, +// proto.ErrUnknownTopicOrPartition is returned. +func (c *OffsetCoordinator) Offset(topic string, partition int32) (offset int64, metadata string, err error) { + if c.OffsetHandler != nil { + return c.OffsetHandler(c.conf.ConsumerGroup, topic, partition) + } + off, ok := c.Offsets[fmt.Sprintf("%s:%d", topic, partition)] + if !ok { + return 0, "", proto.ErrUnknownTopicOrPartition + } + return off, "", nil +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/doc.go b/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/doc.go new file mode 100644 index 0000000000..962a7a2646 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/doc.go @@ -0,0 +1,8 @@ +/* + +Package kafkatest provides mock objects for high level kafka interface. + +Use NewBroker function to create mock broker object and standard methods to create producers and consumers. + +*/ +package kafkatest diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/server.go b/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/server.go new file mode 100644 index 0000000000..c851ec2dcc --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/kafkatest/server.go @@ -0,0 +1,464 @@ +package kafkatest + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "strconv" + "sync" + + "github.com/optiopay/kafka/proto" +) + +type Server struct { + mu sync.RWMutex + brokers []proto.MetadataRespBroker + topics map[string]map[int32][]*proto.Message + ln net.Listener + middlewares []Middleware +} + +// Middleware is function that is called for every incomming kafka message, +// before running default processing handler. Middleware function can return +// nil or kafka response message. +type Middleware func(nodeID int32, requestKind int16, content []byte) Response + +// Response is any kafka response as defined in kafka/proto package +type Response interface { + Bytes() ([]byte, error) +} + +// NewServer return new mock server instance. Any number of middlewares can be +// passed to customize request handling. For every incomming request, all +// middlewares are called one after another in order they were passed. If any +// middleware return non nil response message, response is instasntly written +// to the client and no further code execution for the request is made -- no +// other middleware is called nor the default handler is executed. +func NewServer(middlewares ...Middleware) *Server { + s := &Server{ + brokers: make([]proto.MetadataRespBroker, 0), + topics: make(map[string]map[int32][]*proto.Message), + middlewares: middlewares, + } + return s +} + +// Addr return server instance address or empty string if not running. +func (s *Server) Addr() string { + s.mu.RLock() + defer s.mu.RUnlock() + if s.ln != nil { + return s.ln.Addr().String() + } + return "" +} + +// Reset will clear out local messages and topics. +func (s *Server) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + + s.topics = make(map[string]map[int32][]*proto.Message) +} + +// Close shut down server if running. It is safe to call it more than once. +func (s *Server) Close() (err error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ln != nil { + err = s.ln.Close() + s.ln = nil + } + return err +} + +// ServeHTTP provides JSON serialized server state information. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + topics := make(map[string]map[string][]*proto.Message) + for name, parts := range s.topics { + topics[name] = make(map[string][]*proto.Message) + for part, messages := range parts { + topics[name][strconv.Itoa(int(part))] = messages + } + } + + w.Header().Set("content-type", "application/json") + err := json.NewEncoder(w).Encode(map[string]interface{}{ + "topics": topics, + "brokers": s.brokers, + }) + if err != nil { + log.Printf("cannot JSON encode state: %s", err) + } +} + +// AddMessages append messages to given topic/partition. If topic or partition +// does not exists, it is being created. +// To only create topic/partition, call this method withough giving any +// message. +func (s *Server) AddMessages(topic string, partition int32, messages ...*proto.Message) { + s.mu.Lock() + defer s.mu.Unlock() + + parts, ok := s.topics[topic] + if !ok { + parts = make(map[int32][]*proto.Message) + s.topics[topic] = parts + } + + for i := int32(0); i <= partition; i++ { + if _, ok := parts[i]; !ok { + parts[i] = make([]*proto.Message, 0) + } + } + if len(messages) > 0 { + start := len(parts[partition]) + for i, msg := range messages { + msg.Offset = int64(start + i) + msg.Partition = partition + msg.Topic = topic + } + parts[partition] = append(parts[partition], messages...) + } +} + +// Run starts kafka mock server listening on given address. +func (s *Server) Run(addr string) error { + const nodeID = 100 + + s.mu.RLock() + if s.ln != nil { + s.mu.RUnlock() + log.Printf("server already running: %s", s.ln.Addr()) + return fmt.Errorf("server already running: %s", s.ln.Addr()) + } + + ln, err := net.Listen("tcp4", addr) + if err != nil { + s.mu.RUnlock() + log.Printf("cannot listen on address %q: %s", addr, err) + return fmt.Errorf("cannot listen: %s", err) + } + defer func() { + _ = ln.Close() + }() + s.ln = ln + + if host, port, err := net.SplitHostPort(ln.Addr().String()); err != nil { + s.mu.RUnlock() + log.Printf("cannot extract host/port from %q: %s", ln.Addr(), err) + return fmt.Errorf("cannot extract host/port from %q: %s", ln.Addr(), err) + } else { + prt, err := strconv.Atoi(port) + if err != nil { + s.mu.RUnlock() + log.Printf("invalid port %q: %s", port, err) + return fmt.Errorf("invalid port %q: %s", port, err) + } + s.brokers = append(s.brokers, proto.MetadataRespBroker{ + NodeID: nodeID, + Host: host, + Port: int32(prt), + }) + } + s.mu.RUnlock() + + for { + conn, err := ln.Accept() + if err == nil { + go s.handleClient(nodeID, conn) + } + } +} + +// MustSpawn run server in the background on random port. It panics if server +// cannot be spawned. +// Use Close method to stop spawned server. +func (s *Server) MustSpawn() { + const nodeID = 100 + + s.mu.Lock() + defer s.mu.Unlock() + + if s.ln != nil { + return + } + + ln, err := net.Listen("tcp4", "localhost:0") + if err != nil { + panic(fmt.Sprintf("cannot listen: %s", err)) + } + s.ln = ln + + if host, port, err := net.SplitHostPort(ln.Addr().String()); err != nil { + panic(fmt.Sprintf("cannot extract host/port from %q: %s", ln.Addr(), err)) + } else { + prt, err := strconv.Atoi(port) + if err != nil { + panic(fmt.Sprintf("invalid port %q: %s", port, err)) + } + s.brokers = append(s.brokers, proto.MetadataRespBroker{ + NodeID: nodeID, + Host: host, + Port: int32(prt), + }) + } + + go func() { + for { + conn, err := ln.Accept() + if err == nil { + go s.handleClient(nodeID, conn) + } + } + }() +} + +func (s *Server) handleClient(nodeID int32, conn net.Conn) { + defer func() { + _ = conn.Close() + }() + + for { + kind, b, err := proto.ReadReq(conn) + if err != nil { + if err != io.EOF { + log.Printf("client read error: %s", err) + } + return + } + + var resp response + + for _, middleware := range s.middlewares { + resp = middleware(nodeID, kind, b) + if resp != nil { + break + } + } + + if resp == nil { + switch kind { + case proto.ProduceReqKind: + req, err := proto.ReadProduceReq(bytes.NewBuffer(b)) + if err != nil { + log.Printf("cannot parse produce request: %s\n%s", err, b) + return + } + resp = s.handleProduceRequest(nodeID, conn, req) + case proto.FetchReqKind: + req, err := proto.ReadFetchReq(bytes.NewBuffer(b)) + if err != nil { + log.Printf("cannot parse fetch request: %s\n%s", err, b) + return + } + resp = s.handleFetchRequest(nodeID, conn, req) + case proto.OffsetReqKind: + req, err := proto.ReadOffsetReq(bytes.NewBuffer(b)) + if err != nil { + log.Printf("cannot parse offset request: %s\n%s", err, b) + return + } + resp = s.handleOffsetRequest(nodeID, conn, req) + case proto.MetadataReqKind: + req, err := proto.ReadMetadataReq(bytes.NewBuffer(b)) + if err != nil { + log.Printf("cannot parse metadata request: %s\n%s", err, b) + return + } + resp = s.handleMetadataRequest(nodeID, conn, req) + case proto.OffsetCommitReqKind: + log.Printf("not implemented: %d\n%s", kind, b) + return + case proto.OffsetFetchReqKind: + log.Printf("not implemented: %d\n%s", kind, b) + return + case proto.ConsumerMetadataReqKind: + log.Printf("not implemented: %d\n%s", kind, b) + return + default: + log.Printf("unknown request: %d\n%s", kind, b) + return + } + } + + if resp == nil { + log.Printf("no response for %d", kind) + return + } + b, err = resp.Bytes() + if err != nil { + log.Printf("cannot serialize %T response: %s", resp, err) + } + if _, err := conn.Write(b); err != nil { + log.Printf("cannot write %T response: %s", resp, err) + return + } + } +} + +type response interface { + Bytes() ([]byte, error) +} + +func (s *Server) handleProduceRequest(nodeID int32, conn net.Conn, req *proto.ProduceReq) response { + s.mu.Lock() + defer s.mu.Unlock() + + resp := &proto.ProduceResp{ + CorrelationID: req.CorrelationID, + Topics: make([]proto.ProduceRespTopic, len(req.Topics)), + } + + for ti, topic := range req.Topics { + t, ok := s.topics[topic.Name] + if !ok { + t = make(map[int32][]*proto.Message) + s.topics[topic.Name] = t + } + + respParts := make([]proto.ProduceRespPartition, len(topic.Partitions)) + resp.Topics[ti].Name = topic.Name + resp.Topics[ti].Partitions = respParts + + for pi, part := range topic.Partitions { + p, ok := t[part.ID] + if !ok { + p = make([]*proto.Message, 0) + t[part.ID] = p + } + + for _, msg := range part.Messages { + msg.Offset = int64(len(t[part.ID])) + msg.Topic = topic.Name + t[part.ID] = append(t[part.ID], msg) + } + + respParts[pi].ID = part.ID + respParts[pi].Offset = int64(len(t[part.ID])) - 1 + } + } + return resp +} + +func (s *Server) handleFetchRequest(nodeID int32, conn net.Conn, req *proto.FetchReq) response { + s.mu.RLock() + defer s.mu.RUnlock() + + resp := &proto.FetchResp{ + CorrelationID: req.CorrelationID, + Topics: make([]proto.FetchRespTopic, len(req.Topics)), + } + for ti, topic := range req.Topics { + respParts := make([]proto.FetchRespPartition, len(topic.Partitions)) + resp.Topics[ti].Name = topic.Name + resp.Topics[ti].Partitions = respParts + for pi, part := range topic.Partitions { + respParts[pi].ID = part.ID + + partitions, ok := s.topics[topic.Name] + if !ok { + respParts[pi].Err = proto.ErrUnknownTopicOrPartition + continue + } + messages, ok := partitions[part.ID] + if !ok { + respParts[pi].Err = proto.ErrUnknownTopicOrPartition + continue + } + respParts[pi].TipOffset = int64(len(messages)) + respParts[pi].Messages = messages[part.FetchOffset:] + } + } + + return resp +} + +func (s *Server) handleOffsetRequest(nodeID int32, conn net.Conn, req *proto.OffsetReq) response { + s.mu.RLock() + defer s.mu.RUnlock() + + resp := &proto.OffsetResp{ + CorrelationID: req.CorrelationID, + Topics: make([]proto.OffsetRespTopic, len(req.Topics)), + } + for ti, topic := range req.Topics { + respPart := make([]proto.OffsetRespPartition, len(topic.Partitions)) + resp.Topics[ti].Name = topic.Name + resp.Topics[ti].Partitions = respPart + for pi, part := range topic.Partitions { + respPart[pi].ID = part.ID + switch part.TimeMs { + case -1: // oldest + msgs := len(s.topics[topic.Name][part.ID]) + respPart[pi].Offsets = []int64{int64(msgs), 0} + case -2: // earliest + respPart[pi].Offsets = []int64{0, 0} + default: + log.Printf("offset time for %s:%d not supported: %d", topic.Name, part.ID, part.TimeMs) + return nil + } + } + } + return resp +} + +func (s *Server) handleMetadataRequest(nodeID int32, conn net.Conn, req *proto.MetadataReq) response { + s.mu.RLock() + defer s.mu.RUnlock() + + resp := &proto.MetadataResp{ + CorrelationID: req.CorrelationID, + Topics: make([]proto.MetadataRespTopic, 0, len(s.topics)), + Brokers: s.brokers, + } + + if req.Topics != nil && len(req.Topics) > 0 { + // if particular topic was requested, create empty log if does not yet exists + for _, name := range req.Topics { + partitions, ok := s.topics[name] + if !ok { + partitions = make(map[int32][]*proto.Message) + partitions[0] = make([]*proto.Message, 0) + s.topics[name] = partitions + } + + parts := make([]proto.MetadataRespPartition, len(partitions)) + for pid := range partitions { + p := &parts[pid] + p.ID = pid + p.Leader = nodeID + p.Replicas = []int32{nodeID} + p.Isrs = []int32{nodeID} + } + resp.Topics = append(resp.Topics, proto.MetadataRespTopic{ + Name: name, + Partitions: parts, + }) + + } + } else { + for name, partitions := range s.topics { + parts := make([]proto.MetadataRespPartition, len(partitions)) + for pid := range partitions { + p := &parts[pid] + p.ID = pid + p.Leader = nodeID + p.Replicas = []int32{nodeID} + p.Isrs = []int32{nodeID} + } + resp.Topics = append(resp.Topics, proto.MetadataRespTopic{ + Name: name, + Partitions: parts, + }) + } + } + return resp +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/log.go b/Godeps/_workspace/src/github.com/optiopay/kafka/log.go new file mode 100644 index 0000000000..2ed31970b9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/log.go @@ -0,0 +1,22 @@ +package kafka + +// Logger is general logging interface that can be provided by popular logging +// frameworks. +// +// * https://github.com/go-kit/kit/tree/master/log +// * https://github.com/husio/log +type Logger interface { + Debug(msg string, args ...interface{}) + Info(msg string, args ...interface{}) + Warn(msg string, args ...interface{}) + Error(msg string, args ...interface{}) +} + +// nullLogger implements Logger interface, but discards all messages +type nullLogger struct { +} + +func (nullLogger) Debug(msg string, args ...interface{}) {} +func (nullLogger) Info(msg string, args ...interface{}) {} +func (nullLogger) Warn(msg string, args ...interface{}) {} +func (nullLogger) Error(msg string, args ...interface{}) {} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/multiplexer.go b/Godeps/_workspace/src/github.com/optiopay/kafka/multiplexer.go new file mode 100644 index 0000000000..58cc3abd99 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/multiplexer.go @@ -0,0 +1,122 @@ +package kafka + +import ( + "errors" + "sync" + + "github.com/optiopay/kafka/proto" +) + +// ErrMxClosed is returned as a result of closed multiplexer consumption. +var ErrMxClosed = errors.New("closed") + +// Mx is multiplexer combining into single stream number of consumers. +// +// It is responsibility of the user of the multiplexer and the consumer +// implementation to handle errors. +// ErrNoData returned by consumer is not passed through by the multiplexer, +// instead consumer that returned ErrNoData is removed from merged set. When +// all consumers are removed (set is empty), Mx is automatically closed and any +// further Consume call will result in ErrMxClosed error. +// +// It is important to remember that because fetch from every consumer is done +// by separate worker, most of the time there is one message consumed by each +// worker that is held in memory while waiting for opportunity to return it +// once Consume on multiplexer is called. Closing multiplexer may result in +// ignoring some of already read, waiting for delivery messages kept internally +// by every worker. +type Mx struct { + errc chan error + msgc chan *proto.Message + stop chan struct{} + + mu sync.Mutex + closed bool + workers int +} + +// Merge is merging consume result of any number of consumers into single stream +// and expose them through returned multiplexer. +func Merge(consumers ...Consumer) *Mx { + p := &Mx{ + errc: make(chan error), + msgc: make(chan *proto.Message), + stop: make(chan struct{}), + workers: len(consumers), + } + + for _, consumer := range consumers { + go func(c Consumer) { + defer func() { + p.mu.Lock() + p.workers -= 1 + if p.workers == 0 && !p.closed { + close(p.stop) + p.closed = true + } + p.mu.Unlock() + }() + + for { + msg, err := c.Consume() + if err != nil { + if err == ErrNoData { + return + } + select { + case p.errc <- err: + case <-p.stop: + return + } + } else { + select { + case p.msgc <- msg: + case <-p.stop: + return + } + } + } + }(consumer) + } + + return p +} + +// Workers return number of active consumer workers that are pushing messages +// to multiplexer conumer queue. +func (p *Mx) Workers() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.workers +} + +// Close is closing multiplexer and stopping all underlying workers. +// +// Closing multiplexer will stop all workers as soon as possible, but any +// consume-in-progress action performed by worker has to be finished first. Any +// consumption result received after closing multiplexer is ignored. +// +// Close is returning without waiting for all the workers to finish. +// +// Closing closed multiplexer has no effect. +func (p *Mx) Close() { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.closed { + p.closed = true + close(p.stop) + } +} + +// Consume returns Consume result from any of the merged consumer. +func (p *Mx) Consume() (*proto.Message, error) { + select { + case <-p.stop: + return nil, ErrMxClosed + case msg := <-p.msgc: + return msg, nil + case err := <-p.errc: + return nil, err + } +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/proto/doc.go b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/doc.go new file mode 100644 index 0000000000..ae0ab26d32 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/doc.go @@ -0,0 +1,6 @@ +/* + +Package proto provides kafka binary protocol implementation. + +*/ +package proto diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/proto/errors.go b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/errors.go new file mode 100644 index 0000000000..2f8aa2961c --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/errors.go @@ -0,0 +1,67 @@ +package proto + +import ( + "fmt" +) + +var ( + ErrUnknown = &KafkaError{-1, "unknown"} + ErrOffsetOutOfRange = &KafkaError{1, "offset out of range"} + ErrInvalidMessage = &KafkaError{2, "invalid message"} + ErrUnknownTopicOrPartition = &KafkaError{3, "unknown topic or partition"} + ErrInvalidMessageSize = &KafkaError{4, "invalid message size"} + ErrLeaderNotAvailable = &KafkaError{5, "leader not available"} + ErrNotLeaderForPartition = &KafkaError{6, "not leader for partition"} + ErrRequestTimeout = &KafkaError{7, "request timeed out"} + ErrBrokerNotAvailable = &KafkaError{8, "broker not available"} + ErrReplicaNotAvailable = &KafkaError{9, "replica not available"} + ErrMessageSizeTooLarge = &KafkaError{10, "message size too large"} + ErrScaleControllerEpoch = &KafkaError{11, "scale controller epoch"} + ErrOffsetMetadataTooLarge = &KafkaError{12, "offset metadata too large"} + ErrOffsetLoadInProgress = &KafkaError{14, "offsets load in progress"} + ErrNoCoordinator = &KafkaError{15, "consumer coordinator not available"} + ErrNotCoordinator = &KafkaError{16, "not coordinator for consumer"} + + errnoToErr = map[int16]error{ + -1: ErrUnknown, + 1: ErrOffsetOutOfRange, + 2: ErrInvalidMessage, + 3: ErrUnknownTopicOrPartition, + 4: ErrInvalidMessageSize, + 5: ErrLeaderNotAvailable, + 6: ErrNotLeaderForPartition, + 7: ErrRequestTimeout, + 8: ErrBrokerNotAvailable, + 9: ErrReplicaNotAvailable, + 10: ErrMessageSizeTooLarge, + 11: ErrScaleControllerEpoch, + 12: ErrOffsetMetadataTooLarge, + 14: ErrOffsetLoadInProgress, + 15: ErrNoCoordinator, + 16: ErrNotCoordinator, + } +) + +type KafkaError struct { + errno int16 + message string +} + +func (err *KafkaError) Error() string { + return fmt.Sprintf("%s (%d)", err.message, err.errno) +} + +func (err *KafkaError) Errno() int { + return int(err.errno) +} + +func errFromNo(errno int16) error { + if errno == 0 { + return nil + } + err, ok := errnoToErr[errno] + if !ok { + return fmt.Errorf("unknown kafka error %d", errno) + } + return err +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/proto/messages.go b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/messages.go new file mode 100644 index 0000000000..e34f9de38c --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/messages.go @@ -0,0 +1,1473 @@ +package proto + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + "time" + + "github.com/golang/snappy" +) + +/* + +Kafka wire protocol implemented as described in +https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + +*/ + +const ( + ProduceReqKind = 0 + FetchReqKind = 1 + OffsetReqKind = 2 + MetadataReqKind = 3 + OffsetCommitReqKind = 8 + OffsetFetchReqKind = 9 + ConsumerMetadataReqKind = 10 + + // receive the latest offset (i.e. the offset of the next coming message) + OffsetReqTimeLatest = -1 + + // receive the earliest available offset. Note that because offsets are + // pulled in descending order, asking for the earliest offset will always + // return you a single element. + OffsetReqTimeEarliest = -2 + + // Server will not send any response. + RequiredAcksNone = 0 + + // Server will block until the message is committed by all in sync replicas + // before sending a response. + RequiredAcksAll = -1 + + // Server will wait the data is written to the local log before sending a + // response. + RequiredAcksLocal = 1 +) + +type Compression int8 + +const ( + CompressionNone Compression = 0 + CompressionGzip Compression = 1 + CompressionSnappy Compression = 2 +) + +// ReadReq returns request kind ID and byte representation of the whole message +// in wire protocol format. +func ReadReq(r io.Reader) (requestKind int16, b []byte, err error) { + dec := NewDecoder(r) + msgSize := dec.DecodeInt32() + requestKind = dec.DecodeInt16() + if err := dec.Err(); err != nil { + return 0, nil, err + } + // size of the message + size of the message itself + b = make([]byte, msgSize+4) + binary.BigEndian.PutUint32(b, uint32(msgSize)) + binary.BigEndian.PutUint16(b[4:], uint16(requestKind)) + if _, err := io.ReadFull(r, b[6:]); err != nil { + return 0, nil, err + } + return requestKind, b, err +} + +// ReadResp returns message correlation ID and byte representation of the whole +// message in wire protocol that is returned when reading from given stream, +// including 4 bytes of message size itself. +// Byte representation returned by ReadResp can be parsed by all response +// reeaders to transform it into specialized response structure. +func ReadResp(r io.Reader) (correlationID int32, b []byte, err error) { + dec := NewDecoder(r) + msgSize := dec.DecodeInt32() + correlationID = dec.DecodeInt32() + if err := dec.Err(); err != nil { + return 0, nil, err + } + // size of the message + size of the message itself + b = make([]byte, msgSize+4) + binary.BigEndian.PutUint32(b, uint32(msgSize)) + binary.BigEndian.PutUint32(b[4:], uint32(correlationID)) + _, err = io.ReadFull(r, b[8:]) + return correlationID, b, err +} + +// Message represents single entity of message set. +type Message struct { + Key []byte + Value []byte + Offset int64 // set when fetching and after successful producing + Crc uint32 // set when fetching, ignored when producing + Topic string // set when fetching, ignored when producing + Partition int32 // set when fetching, ignored when producing + TipOffset int64 // set when fetching, ignored when processing +} + +// ComputeCrc returns crc32 hash for given message content. +func ComputeCrc(m *Message, compression Compression) uint32 { + var buf bytes.Buffer + enc := NewEncoder(&buf) + enc.EncodeInt8(0) // magic byte is always 0 + enc.EncodeInt8(int8(compression)) + enc.EncodeBytes(m.Key) + enc.EncodeBytes(m.Value) + return crc32.ChecksumIEEE(buf.Bytes()) +} + +// writeMessageSet writes a Message Set into w. +// It returns the number of bytes written and any error. +func writeMessageSet(w io.Writer, messages []*Message, compression Compression) (int, error) { + if len(messages) == 0 { + return 0, nil + } + // NOTE(caleb): it doesn't appear to be documented, but I observed that the + // Java client sets the offset of the synthesized message set for a group of + // compressed messages to be the offset of the last message in the set. + compressOffset := messages[len(messages)-1].Offset + switch compression { + case CompressionGzip: + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := writeMessageSet(gz, messages, CompressionNone); err != nil { + return 0, err + } + if err := gz.Close(); err != nil { + return 0, err + } + messages = []*Message{ + { + Value: buf.Bytes(), + Offset: compressOffset, + }, + } + case CompressionSnappy: + var buf bytes.Buffer + if _, err := writeMessageSet(&buf, messages, CompressionNone); err != nil { + return 0, err + } + messages = []*Message{ + { + Value: snappy.Encode(nil, buf.Bytes()), + Offset: compressOffset, + }, + } + } + + totalSize := 0 + b := newSliceWriter(0) + for _, message := range messages { + bsize := 26 + len(message.Key) + len(message.Value) + b.Reset(bsize) + + enc := NewEncoder(b) + enc.EncodeInt64(message.Offset) + msize := int32(14 + len(message.Key) + len(message.Value)) + enc.EncodeInt32(msize) + enc.EncodeUint32(0) // crc32 placeholder + enc.EncodeInt8(0) // magic byte + enc.EncodeInt8(int8(compression)) + enc.EncodeBytes(message.Key) + enc.EncodeBytes(message.Value) + + if err := enc.Err(); err != nil { + return totalSize, err + } + + const hsize = 8 + 4 + 4 // offset + message size + crc32 + const crcoff = 8 + 4 // offset + message size + binary.BigEndian.PutUint32(b.buf[crcoff:crcoff+4], crc32.ChecksumIEEE(b.buf[hsize:bsize])) + + if n, err := w.Write(b.Slice()); err != nil { + return totalSize, err + } else { + totalSize += n + } + + } + return totalSize, nil +} + +type slicewriter struct { + buf []byte + pos int + size int +} + +func newSliceWriter(bufsize int) *slicewriter { + return &slicewriter{ + buf: make([]byte, bufsize), + pos: 0, + } +} + +func (w *slicewriter) Write(p []byte) (int, error) { + if len(w.buf) < w.pos+len(p) { + return 0, errors.New("buffer too small") + } + copy(w.buf[w.pos:], p) + w.pos += len(p) + return len(p), nil +} + +func (w *slicewriter) Reset(size int) { + if size > len(w.buf) { + w.buf = make([]byte, size+1000) // allocate a bit more than required + } + w.size = size + w.pos = 0 +} + +func (w *slicewriter) Slice() []byte { + return w.buf[:w.pos] +} + +// readMessageSet reads and return messages from the stream. +// The size is known before a message set is decoded. +// Because kafka is sending message set directly from the drive, it might cut +// off part of the last message. This also means that the last message can be +// shorter than the header is saying. In such case just ignore the last +// malformed message from the set and returned earlier data. +func readMessageSet(r io.Reader, size int32) ([]*Message, error) { + rd := io.LimitReader(r, int64(size)) + dec := NewDecoder(rd) + set := make([]*Message, 0, 256) + + var buf []byte + for { + offset := dec.DecodeInt64() + if err := dec.Err(); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return set, nil + } + return nil, err + } + // single message size + size := dec.DecodeInt32() + if err := dec.Err(); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return set, nil + } + return nil, err + } + + // read message to buffer to compute its content crc + if int(size) > len(buf) { + // allocate a bit more than needed + buf = make([]byte, size+10240) + } + msgbuf := buf[:size] + + if _, err := io.ReadFull(rd, msgbuf); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return set, nil + } + return nil, err + } + msgdec := NewDecoder(bytes.NewBuffer(msgbuf)) + + msg := &Message{ + Offset: offset, + Crc: msgdec.DecodeUint32(), + } + + if msg.Crc != crc32.ChecksumIEEE(msgbuf[4:]) { + // ignore this message and because we want to have constant + // history, do not process anything more + return set, nil + } + + // magic byte + _ = msgdec.DecodeInt8() + + attributes := msgdec.DecodeInt8() + switch compression := Compression(attributes & 3); compression { + case CompressionNone: + msg.Key = msgdec.DecodeBytes() + msg.Value = msgdec.DecodeBytes() + if err := msgdec.Err(); err != nil { + return nil, fmt.Errorf("cannot decode message: %s", err) + } + set = append(set, msg) + case CompressionGzip, CompressionSnappy: + _ = msgdec.DecodeBytes() // ignore key + val := msgdec.DecodeBytes() + if err := msgdec.Err(); err != nil { + return nil, fmt.Errorf("cannot decode message: %s", err) + } + var decoded []byte + switch compression { + case CompressionGzip: + cr, err := gzip.NewReader(bytes.NewReader(val)) + if err != nil { + return nil, fmt.Errorf("error decoding gzip message: %s", err) + } + decoded, err = ioutil.ReadAll(cr) + if err != nil { + return nil, fmt.Errorf("error decoding gzip message: %s", err) + } + _ = cr.Close() + case CompressionSnappy: + var err error + decoded, err = snappyDecode(val) + if err != nil { + return nil, fmt.Errorf("error decoding snappy message: %s", err) + } + } + msgs, err := readMessageSet(bytes.NewReader(decoded), int32(len(decoded))) + if err != nil { + return nil, err + } + set = append(set, msgs...) + default: + return nil, fmt.Errorf("cannot handle compression method: %d", compression) + } + } +} + +type MetadataReq struct { + CorrelationID int32 + ClientID string + Topics []string +} + +func ReadMetadataReq(r io.Reader) (*MetadataReq, error) { + var req MetadataReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + req.Topics = make([]string, dec.DecodeArrayLen()) + for i := range req.Topics { + req.Topics[i] = dec.DecodeString() + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *MetadataReq) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(int16(MetadataReqKind)) + enc.Encode(int16(0)) + enc.Encode(r.CorrelationID) + enc.Encode(r.ClientID) + + enc.EncodeArrayLen(len(r.Topics)) + for _, name := range r.Topics { + enc.Encode(name) + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func (r *MetadataReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type MetadataResp struct { + CorrelationID int32 + Brokers []MetadataRespBroker + Topics []MetadataRespTopic +} + +type MetadataRespBroker struct { + NodeID int32 + Host string + Port int32 +} + +type MetadataRespTopic struct { + Name string + Err error + Partitions []MetadataRespPartition +} + +type MetadataRespPartition struct { + ID int32 + Err error + Leader int32 + Replicas []int32 + Isrs []int32 +} + +func (r *MetadataResp) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(r.CorrelationID) + enc.EncodeArrayLen(len(r.Brokers)) + for _, broker := range r.Brokers { + enc.Encode(broker.NodeID) + enc.Encode(broker.Host) + enc.Encode(broker.Port) + } + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.EncodeError(topic.Err) + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.EncodeError(part.Err) + enc.Encode(part.ID) + enc.Encode(part.Leader) + enc.Encode(part.Replicas) + enc.Encode(part.Isrs) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func ReadMetadataResp(r io.Reader) (*MetadataResp, error) { + var resp MetadataResp + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + + resp.Brokers = make([]MetadataRespBroker, dec.DecodeArrayLen()) + for i := range resp.Brokers { + var b = &resp.Brokers[i] + b.NodeID = dec.DecodeInt32() + b.Host = dec.DecodeString() + b.Port = dec.DecodeInt32() + } + + resp.Topics = make([]MetadataRespTopic, dec.DecodeArrayLen()) + for ti := range resp.Topics { + var t = &resp.Topics[ti] + t.Err = errFromNo(dec.DecodeInt16()) + t.Name = dec.DecodeString() + t.Partitions = make([]MetadataRespPartition, dec.DecodeArrayLen()) + for pi := range t.Partitions { + var p = &t.Partitions[pi] + p.Err = errFromNo(dec.DecodeInt16()) + p.ID = dec.DecodeInt32() + p.Leader = dec.DecodeInt32() + + p.Replicas = make([]int32, dec.DecodeArrayLen()) + for ri := range p.Replicas { + p.Replicas[ri] = dec.DecodeInt32() + } + + p.Isrs = make([]int32, dec.DecodeArrayLen()) + for ii := range p.Isrs { + p.Isrs[ii] = dec.DecodeInt32() + } + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &resp, nil +} + +type FetchReq struct { + CorrelationID int32 + ClientID string + MaxWaitTime time.Duration + MinBytes int32 + + Topics []FetchReqTopic +} + +type FetchReqTopic struct { + Name string + Partitions []FetchReqPartition +} + +type FetchReqPartition struct { + ID int32 + FetchOffset int64 + MaxBytes int32 +} + +func ReadFetchReq(r io.Reader) (*FetchReq, error) { + var req FetchReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + // replica id + _ = dec.DecodeInt32() + req.MaxWaitTime = time.Duration(dec.DecodeInt32()) * time.Millisecond + req.MinBytes = dec.DecodeInt32() + req.Topics = make([]FetchReqTopic, dec.DecodeArrayLen()) + for ti := range req.Topics { + var topic = &req.Topics[ti] + topic.Name = dec.DecodeString() + topic.Partitions = make([]FetchReqPartition, dec.DecodeArrayLen()) + for pi := range topic.Partitions { + var part = &topic.Partitions[pi] + part.ID = dec.DecodeInt32() + part.FetchOffset = dec.DecodeInt64() + part.MaxBytes = dec.DecodeInt32() + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *FetchReq) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(int16(FetchReqKind)) + enc.Encode(int16(0)) + enc.Encode(r.CorrelationID) + enc.Encode(r.ClientID) + + // replica id + enc.Encode(int32(-1)) + enc.Encode(int32(r.MaxWaitTime / time.Millisecond)) + enc.Encode(r.MinBytes) + + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.Encode(part.FetchOffset) + enc.Encode(part.MaxBytes) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func (r *FetchReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type FetchResp struct { + CorrelationID int32 + Topics []FetchRespTopic +} + +type FetchRespTopic struct { + Name string + Partitions []FetchRespPartition +} + +type FetchRespPartition struct { + ID int32 + Err error + TipOffset int64 + Messages []*Message +} + +func (r *FetchResp) Bytes() ([]byte, error) { + var buf buffer + enc := NewEncoder(&buf) + + enc.Encode(int32(0)) // placeholder + enc.Encode(r.CorrelationID) + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.EncodeError(part.Err) + enc.Encode(part.TipOffset) + i := len(buf) + enc.Encode(int32(0)) // placeholder + // NOTE(caleb): writing compressed fetch response isn't implemented + // for now, since that's not needed for clients. + n, err := writeMessageSet(&buf, part.Messages, CompressionNone) + if err != nil { + return nil, err + } + binary.BigEndian.PutUint32(buf[i:i+4], uint32(n)) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + binary.BigEndian.PutUint32(buf[:4], uint32(len(buf)-4)) + return []byte(buf), nil +} + +func ReadFetchResp(r io.Reader) (*FetchResp, error) { + var err error + var resp FetchResp + + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + + resp.Topics = make([]FetchRespTopic, dec.DecodeArrayLen()) + for ti := range resp.Topics { + var topic = &resp.Topics[ti] + topic.Name = dec.DecodeString() + topic.Partitions = make([]FetchRespPartition, dec.DecodeArrayLen()) + for pi := range topic.Partitions { + var part = &topic.Partitions[pi] + part.ID = dec.DecodeInt32() + part.Err = errFromNo(dec.DecodeInt16()) + part.TipOffset = dec.DecodeInt64() + if dec.Err() != nil { + return nil, dec.Err() + } + msgSetSize := dec.DecodeInt32() + if dec.Err() != nil { + return nil, dec.Err() + } + if part.Messages, err = readMessageSet(r, msgSetSize); err != nil { + return nil, err + } + for _, msg := range part.Messages { + msg.Topic = topic.Name + msg.Partition = part.ID + msg.TipOffset = part.TipOffset + } + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &resp, nil +} + +type ConsumerMetadataReq struct { + CorrelationID int32 + ClientID string + ConsumerGroup string +} + +func ReadConsumerMetadataReq(r io.Reader) (*ConsumerMetadataReq, error) { + var req ConsumerMetadataReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + req.ConsumerGroup = dec.DecodeString() + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *ConsumerMetadataReq) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(int16(ConsumerMetadataReqKind)) + enc.Encode(int16(0)) + enc.Encode(r.CorrelationID) + enc.Encode(r.ClientID) + + enc.Encode(r.ConsumerGroup) + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func (r *ConsumerMetadataReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type ConsumerMetadataResp struct { + CorrelationID int32 + Err error + CoordinatorID int32 + CoordinatorHost string + CoordinatorPort int32 +} + +func ReadConsumerMetadataResp(r io.Reader) (*ConsumerMetadataResp, error) { + var resp ConsumerMetadataResp + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + resp.Err = errFromNo(dec.DecodeInt16()) + resp.CoordinatorID = dec.DecodeInt32() + resp.CoordinatorHost = dec.DecodeString() + resp.CoordinatorPort = dec.DecodeInt32() + + if err := dec.Err(); err != nil { + return nil, err + } + return &resp, nil +} + +func (r *ConsumerMetadataResp) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(r.CorrelationID) + enc.EncodeError(r.Err) + enc.Encode(r.CoordinatorID) + enc.Encode(r.CoordinatorHost) + enc.Encode(r.CoordinatorPort) + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +type OffsetCommitReq struct { + CorrelationID int32 + ClientID string + ConsumerGroup string + Topics []OffsetCommitReqTopic +} + +type OffsetCommitReqTopic struct { + Name string + Partitions []OffsetCommitReqPartition +} + +type OffsetCommitReqPartition struct { + ID int32 + Offset int64 + TimeStamp time.Time + Metadata string +} + +func ReadOffsetCommitReq(r io.Reader) (*OffsetCommitReq, error) { + var req OffsetCommitReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + req.ConsumerGroup = dec.DecodeString() + req.Topics = make([]OffsetCommitReqTopic, dec.DecodeArrayLen()) + for ti := range req.Topics { + var topic = &req.Topics[ti] + topic.Name = dec.DecodeString() + topic.Partitions = make([]OffsetCommitReqPartition, dec.DecodeArrayLen()) + for pi := range topic.Partitions { + var part = &topic.Partitions[pi] + part.ID = dec.DecodeInt32() + part.Offset = dec.DecodeInt64() + part.TimeStamp = time.Unix(0, dec.DecodeInt64()*int64(time.Millisecond)) + part.Metadata = dec.DecodeString() + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *OffsetCommitReq) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(int16(OffsetCommitReqKind)) + enc.Encode(int16(0)) + enc.Encode(r.CorrelationID) + enc.Encode(r.ClientID) + + enc.Encode(r.ConsumerGroup) + + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.Encode(part.Offset) + // TODO(husio) is this really in milliseconds? + enc.Encode(part.TimeStamp.UnixNano() / int64(time.Millisecond)) + enc.Encode(part.Metadata) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func (r *OffsetCommitReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type OffsetCommitResp struct { + CorrelationID int32 + Topics []OffsetCommitRespTopic +} + +type OffsetCommitRespTopic struct { + Name string + Partitions []OffsetCommitRespPartition +} + +type OffsetCommitRespPartition struct { + ID int32 + Err error +} + +func ReadOffsetCommitResp(r io.Reader) (*OffsetCommitResp, error) { + var resp OffsetCommitResp + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + resp.Topics = make([]OffsetCommitRespTopic, dec.DecodeArrayLen()) + for ti := range resp.Topics { + var t = &resp.Topics[ti] + t.Name = dec.DecodeString() + t.Partitions = make([]OffsetCommitRespPartition, dec.DecodeArrayLen()) + for pi := range t.Partitions { + var p = &t.Partitions[pi] + p.ID = dec.DecodeInt32() + p.Err = errFromNo(dec.DecodeInt16()) + } + } + + if err := dec.Err(); err != nil { + return nil, err + } + return &resp, nil +} + +func (r *OffsetCommitResp) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(r.CorrelationID) + enc.EncodeArrayLen(len(r.Topics)) + for _, t := range r.Topics { + enc.Encode(t.Name) + enc.EncodeArrayLen(len(t.Partitions)) + for _, p := range t.Partitions { + enc.Encode(p.ID) + enc.EncodeError(p.Err) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil + +} + +type OffsetFetchReq struct { + CorrelationID int32 + ClientID string + ConsumerGroup string + Topics []OffsetFetchReqTopic +} + +type OffsetFetchReqTopic struct { + Name string + Partitions []int32 +} + +func ReadOffsetFetchReq(r io.Reader) (*OffsetFetchReq, error) { + var req OffsetFetchReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + req.ConsumerGroup = dec.DecodeString() + req.Topics = make([]OffsetFetchReqTopic, dec.DecodeArrayLen()) + for ti := range req.Topics { + var topic = &req.Topics[ti] + topic.Name = dec.DecodeString() + topic.Partitions = make([]int32, dec.DecodeArrayLen()) + for pi := range topic.Partitions { + topic.Partitions[pi] = dec.DecodeInt32() + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *OffsetFetchReq) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(int16(OffsetFetchReqKind)) + enc.Encode(int16(0)) + enc.Encode(r.CorrelationID) + enc.Encode(r.ClientID) + + enc.Encode(r.ConsumerGroup) + enc.EncodeArrayLen(len(r.Topics)) + for _, t := range r.Topics { + enc.Encode(t.Name) + enc.EncodeArrayLen(len(t.Partitions)) + for _, p := range t.Partitions { + enc.Encode(p) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func (r *OffsetFetchReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type OffsetFetchResp struct { + CorrelationID int32 + Topics []OffsetFetchRespTopic +} + +type OffsetFetchRespTopic struct { + Name string + Partitions []OffsetFetchRespPartition +} + +type OffsetFetchRespPartition struct { + ID int32 + Offset int64 + Metadata string + Err error +} + +func ReadOffsetFetchResp(r io.Reader) (*OffsetFetchResp, error) { + var resp OffsetFetchResp + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + resp.Topics = make([]OffsetFetchRespTopic, dec.DecodeArrayLen()) + for ti := range resp.Topics { + var t = &resp.Topics[ti] + t.Name = dec.DecodeString() + t.Partitions = make([]OffsetFetchRespPartition, dec.DecodeArrayLen()) + for pi := range t.Partitions { + var p = &t.Partitions[pi] + p.ID = dec.DecodeInt32() + p.Offset = dec.DecodeInt64() + p.Metadata = dec.DecodeString() + p.Err = errFromNo(dec.DecodeInt16()) + } + } + + if err := dec.Err(); err != nil { + return nil, err + } + return &resp, nil +} + +func (r *OffsetFetchResp) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(r.CorrelationID) + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.Encode(part.Offset) + enc.Encode(part.Metadata) + enc.EncodeError(part.Err) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +type ProduceReq struct { + CorrelationID int32 + ClientID string + Compression Compression // only used when sending ProduceReqs + RequiredAcks int16 + Timeout time.Duration + Topics []ProduceReqTopic +} + +type ProduceReqTopic struct { + Name string + Partitions []ProduceReqPartition +} + +type ProduceReqPartition struct { + ID int32 + Messages []*Message +} + +func ReadProduceReq(r io.Reader) (*ProduceReq, error) { + var req ProduceReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + req.RequiredAcks = dec.DecodeInt16() + req.Timeout = time.Duration(dec.DecodeInt32()) * time.Millisecond + req.Topics = make([]ProduceReqTopic, dec.DecodeArrayLen()) + for ti := range req.Topics { + var topic = &req.Topics[ti] + topic.Name = dec.DecodeString() + topic.Partitions = make([]ProduceReqPartition, dec.DecodeArrayLen()) + for pi := range topic.Partitions { + var part = &topic.Partitions[pi] + part.ID = dec.DecodeInt32() + if dec.Err() != nil { + return nil, dec.Err() + } + msgSetSize := dec.DecodeInt32() + if dec.Err() != nil { + return nil, dec.Err() + } + var err error + if part.Messages, err = readMessageSet(r, msgSetSize); err != nil { + return nil, err + } + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *ProduceReq) Bytes() ([]byte, error) { + var buf buffer + enc := NewEncoder(&buf) + + enc.EncodeInt32(0) // placeholder + enc.EncodeInt16(ProduceReqKind) + enc.EncodeInt16(0) + enc.EncodeInt32(r.CorrelationID) + enc.EncodeString(r.ClientID) + + enc.EncodeInt16(r.RequiredAcks) + enc.EncodeInt32(int32(r.Timeout / time.Millisecond)) + enc.EncodeArrayLen(len(r.Topics)) + for _, t := range r.Topics { + enc.EncodeString(t.Name) + enc.EncodeArrayLen(len(t.Partitions)) + for _, p := range t.Partitions { + enc.EncodeInt32(p.ID) + i := len(buf) + enc.EncodeInt32(0) // placeholder + n, err := writeMessageSet(&buf, p.Messages, r.Compression) + if err != nil { + return nil, err + } + binary.BigEndian.PutUint32(buf[i:i+4], uint32(n)) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + binary.BigEndian.PutUint32(buf[0:4], uint32(len(buf)-4)) + return []byte(buf), nil +} + +func (r *ProduceReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type ProduceResp struct { + CorrelationID int32 + Topics []ProduceRespTopic +} + +type ProduceRespTopic struct { + Name string + Partitions []ProduceRespPartition +} + +type ProduceRespPartition struct { + ID int32 + Err error + Offset int64 +} + +func (r *ProduceResp) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(r.CorrelationID) + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.EncodeError(part.Err) + enc.Encode(part.Offset) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func ReadProduceResp(r io.Reader) (*ProduceResp, error) { + var resp ProduceResp + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + resp.Topics = make([]ProduceRespTopic, dec.DecodeArrayLen()) + for ti := range resp.Topics { + var t = &resp.Topics[ti] + t.Name = dec.DecodeString() + t.Partitions = make([]ProduceRespPartition, dec.DecodeArrayLen()) + for pi := range t.Partitions { + var p = &t.Partitions[pi] + p.ID = dec.DecodeInt32() + p.Err = errFromNo(dec.DecodeInt16()) + p.Offset = dec.DecodeInt64() + } + } + + if err := dec.Err(); err != nil { + return nil, err + } + return &resp, nil +} + +type OffsetReq struct { + CorrelationID int32 + ClientID string + ReplicaID int32 + Topics []OffsetReqTopic +} + +type OffsetReqTopic struct { + Name string + Partitions []OffsetReqPartition +} + +type OffsetReqPartition struct { + ID int32 + TimeMs int64 // cannot be time.Time because of negative values + MaxOffsets int32 +} + +func ReadOffsetReq(r io.Reader) (*OffsetReq, error) { + var req OffsetReq + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + // api key + api version + _ = dec.DecodeInt32() + req.CorrelationID = dec.DecodeInt32() + req.ClientID = dec.DecodeString() + req.ReplicaID = dec.DecodeInt32() + req.Topics = make([]OffsetReqTopic, dec.DecodeArrayLen()) + for ti := range req.Topics { + var topic = &req.Topics[ti] + topic.Name = dec.DecodeString() + topic.Partitions = make([]OffsetReqPartition, dec.DecodeArrayLen()) + for pi := range topic.Partitions { + var part = &topic.Partitions[pi] + part.ID = dec.DecodeInt32() + part.TimeMs = dec.DecodeInt64() + part.MaxOffsets = dec.DecodeInt32() + } + } + + if dec.Err() != nil { + return nil, dec.Err() + } + return &req, nil +} + +func (r *OffsetReq) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(int16(OffsetReqKind)) + enc.Encode(int16(0)) + enc.Encode(r.CorrelationID) + enc.Encode(r.ClientID) + + enc.Encode(r.ReplicaID) + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.Encode(part.TimeMs) + enc.Encode(part.MaxOffsets) + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +func (r *OffsetReq) WriteTo(w io.Writer) (int64, error) { + b, err := r.Bytes() + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +type OffsetResp struct { + CorrelationID int32 + Topics []OffsetRespTopic +} + +type OffsetRespTopic struct { + Name string + Partitions []OffsetRespPartition +} + +type OffsetRespPartition struct { + ID int32 + Err error + Offsets []int64 +} + +func ReadOffsetResp(r io.Reader) (*OffsetResp, error) { + var resp OffsetResp + dec := NewDecoder(r) + + // total message size + _ = dec.DecodeInt32() + resp.CorrelationID = dec.DecodeInt32() + resp.Topics = make([]OffsetRespTopic, dec.DecodeArrayLen()) + for ti := range resp.Topics { + var t = &resp.Topics[ti] + t.Name = dec.DecodeString() + t.Partitions = make([]OffsetRespPartition, dec.DecodeArrayLen()) + for pi := range t.Partitions { + var p = &t.Partitions[pi] + p.ID = dec.DecodeInt32() + p.Err = errFromNo(dec.DecodeInt16()) + p.Offsets = make([]int64, dec.DecodeArrayLen()) + for oi := range p.Offsets { + p.Offsets[oi] = dec.DecodeInt64() + } + } + } + + if err := dec.Err(); err != nil { + return nil, err + } + return &resp, nil +} + +func (r *OffsetResp) Bytes() ([]byte, error) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + // message size - for now just placeholder + enc.Encode(int32(0)) + enc.Encode(r.CorrelationID) + enc.EncodeArrayLen(len(r.Topics)) + for _, topic := range r.Topics { + enc.Encode(topic.Name) + enc.EncodeArrayLen(len(topic.Partitions)) + for _, part := range topic.Partitions { + enc.Encode(part.ID) + enc.EncodeError(part.Err) + enc.EncodeArrayLen(len(part.Offsets)) + for _, off := range part.Offsets { + enc.Encode(off) + } + } + } + + if enc.Err() != nil { + return nil, enc.Err() + } + + // update the message size information + b := buf.Bytes() + binary.BigEndian.PutUint32(b, uint32(len(b)-4)) + + return b, nil +} + +type buffer []byte + +func (b *buffer) Write(p []byte) (int, error) { + *b = append(*b, p...) + return len(p), nil +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/proto/serialization.go b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/serialization.go new file mode 100644 index 0000000000..8858f5cd3a --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/serialization.go @@ -0,0 +1,364 @@ +package proto + +import ( + "encoding/binary" + "errors" + "fmt" + "io" +) + +var ErrNotEnoughData = errors.New("not enough data") + +type decoder struct { + buf []byte + r io.Reader + err error +} + +func NewDecoder(r io.Reader) *decoder { + return &decoder{ + r: r, + buf: make([]byte, 1024), + } +} + +func (d *decoder) DecodeInt8() int8 { + if d.err != nil { + return 0 + } + b := d.buf[:1] + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return 0 + } + if n != 1 { + d.err = ErrNotEnoughData + return 0 + } + return int8(b[0]) +} + +func (d *decoder) DecodeInt16() int16 { + if d.err != nil { + return 0 + } + b := d.buf[:2] + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return 0 + } + if n != 2 { + d.err = ErrNotEnoughData + return 0 + } + return int16(binary.BigEndian.Uint16(b)) +} + +func (d *decoder) DecodeInt32() int32 { + if d.err != nil { + return 0 + } + b := d.buf[:4] + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return 0 + } + if n != 4 { + d.err = ErrNotEnoughData + return 0 + } + return int32(binary.BigEndian.Uint32(b)) +} + +func (d *decoder) DecodeUint32() uint32 { + if d.err != nil { + return 0 + } + b := d.buf[:4] + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return 0 + } + if n != 4 { + d.err = ErrNotEnoughData + return 0 + } + return binary.BigEndian.Uint32(b) +} + +func (d *decoder) DecodeInt64() int64 { + if d.err != nil { + return 0 + } + b := d.buf[:8] + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return 0 + } + if n != 8 { + d.err = ErrNotEnoughData + return 0 + } + return int64(binary.BigEndian.Uint64(b)) +} + +func (d *decoder) DecodeString() string { + if d.err != nil { + return "" + } + slen := d.DecodeInt16() + if d.err != nil { + return "" + } + if slen < 1 { + return "" + } + + var b []byte + if int(slen) > len(d.buf) { + b = make([]byte, slen) + } else { + b = d.buf[:int(slen)] + } + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return "" + } + if n != int(slen) { + d.err = ErrNotEnoughData + return "" + } + return string(b) +} + +func (d *decoder) DecodeArrayLen() int { + return int(d.DecodeInt32()) +} + +func (d *decoder) DecodeBytes() []byte { + if d.err != nil { + return nil + } + slen := d.DecodeInt32() + if d.err != nil { + return nil + } + if slen < 1 { + return nil + } + + b := make([]byte, slen) + n, err := io.ReadFull(d.r, b) + if err != nil { + d.err = err + return nil + } + if n != int(slen) { + d.err = ErrNotEnoughData + return nil + } + return b +} + +func (d *decoder) Err() error { + return d.err +} + +type encoder struct { + w io.Writer + err error + buf [8]byte +} + +func NewEncoder(w io.Writer) *encoder { + return &encoder{w: w} +} + +func (e *encoder) Encode(value interface{}) { + if e.err != nil { + return + } + var b []byte + + switch val := value.(type) { + case int8: + _, e.err = e.w.Write([]byte{byte(val)}) + case int16: + b = e.buf[:2] + binary.BigEndian.PutUint16(b, uint16(val)) + case int32: + b = e.buf[:4] + binary.BigEndian.PutUint32(b, uint32(val)) + case int64: + b = e.buf[:8] + binary.BigEndian.PutUint64(b, uint64(val)) + case uint16: + b = e.buf[:2] + binary.BigEndian.PutUint16(b, val) + case uint32: + b = e.buf[:4] + binary.BigEndian.PutUint32(b, val) + case uint64: + b = e.buf[:8] + binary.BigEndian.PutUint64(b, val) + case string: + buf := e.buf[:2] + binary.BigEndian.PutUint16(buf, uint16(len(val))) + e.err = writeAll(e.w, buf) + if e.err == nil { + e.err = writeAll(e.w, []byte(val)) + } + case []byte: + buf := e.buf[:4] + + if val == nil { + no := int32(-1) + binary.BigEndian.PutUint32(buf, uint32(no)) + e.err = writeAll(e.w, buf) + return + } + + binary.BigEndian.PutUint32(buf, uint32(len(val))) + e.err = writeAll(e.w, buf) + if e.err == nil { + e.err = writeAll(e.w, val) + } + case []int32: + e.EncodeArrayLen(len(val)) + for _, v := range val { + e.Encode(v) + } + default: + e.err = fmt.Errorf("cannot encode type %T", value) + } + + if b != nil { + e.err = writeAll(e.w, b) + return + } +} + +func (e *encoder) EncodeInt8(val int8) { + if e.err != nil { + return + } + + _, e.err = e.w.Write([]byte{byte(val)}) +} + +func (e *encoder) EncodeInt16(val int16) { + if e.err != nil { + return + } + + b := e.buf[:2] + binary.BigEndian.PutUint16(b, uint16(val)) + e.err = writeAll(e.w, b) +} + +func (e *encoder) EncodeInt32(val int32) { + if e.err != nil { + return + } + + b := e.buf[:4] + binary.BigEndian.PutUint32(b, uint32(val)) + e.err = writeAll(e.w, b) +} + +func (e *encoder) EncodeInt64(val int64) { + if e.err != nil { + return + } + + b := e.buf[:8] + binary.BigEndian.PutUint64(b, uint64(val)) + e.err = writeAll(e.w, b) +} + +func (e *encoder) EncodeUint32(val uint32) { + if e.err != nil { + return + } + + b := e.buf[:4] + binary.BigEndian.PutUint32(b, val) + e.err = writeAll(e.w, b) +} + +func (e *encoder) EncodeBytes(val []byte) { + if e.err != nil { + return + } + + buf := e.buf[:4] + + if val == nil { + no := int32(-1) + binary.BigEndian.PutUint32(buf, uint32(no)) + e.err = writeAll(e.w, buf) + return + } + + binary.BigEndian.PutUint32(buf, uint32(len(val))) + e.err = writeAll(e.w, buf) + if e.err == nil { + e.err = writeAll(e.w, val) + } +} + +func (e *encoder) EncodeString(val string) { + if e.err != nil { + return + } + + buf := e.buf[:2] + + binary.BigEndian.PutUint16(buf, uint16(len(val))) + e.err = writeAll(e.w, buf) + if e.err == nil { + e.err = writeAll(e.w, []byte(val)) + } +} + +func (e *encoder) EncodeError(err error) { + b := e.buf[:2] + + if err == nil { + binary.BigEndian.PutUint16(b, uint16(0)) + e.err = writeAll(e.w, b) + return + } + kerr, ok := err.(*KafkaError) + if !ok { + e.err = fmt.Errorf("cannot encode error of type %T", err) + } + + binary.BigEndian.PutUint16(b, uint16(kerr.errno)) + e.err = writeAll(e.w, b) +} + +func (e *encoder) EncodeArrayLen(length int) { + e.EncodeInt32(int32(length)) +} + +func (e *encoder) Err() error { + return e.err +} + +func writeAll(w io.Writer, b []byte) error { + n, err := w.Write(b) + if err != nil { + return err + } + if n != len(b) { + return fmt.Errorf("cannot write %d: %d written", len(b), n) + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/optiopay/kafka/proto/snappy.go b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/snappy.go new file mode 100644 index 0000000000..f000cc3423 --- /dev/null +++ b/Godeps/_workspace/src/github.com/optiopay/kafka/proto/snappy.go @@ -0,0 +1,50 @@ +package proto + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/golang/snappy" +) + +// Snappy-encoded messages from the official Java client are encoded using +// snappy-java: see github.com/xerial/snappy-java. +// This does its own non-standard framing. We can detect this encoding +// by sniffing its special header. +// +// That library will still read plain (unframed) snappy-encoded messages, +// so we don't need to implement that codec on the compression side. +// +// (This is the same behavior as several of the other popular Kafka clients.) + +var snappyJavaMagic = []byte("\x82SNAPPY\x00") + +func snappyDecode(b []byte) ([]byte, error) { + if !bytes.HasPrefix(b, snappyJavaMagic) { + return snappy.Decode(nil, b) + } + + // See https://github.com/xerial/snappy-java/blob/develop/src/main/java/org/xerial/snappy/SnappyInputStream.java + version := binary.BigEndian.Uint32(b[8:12]) + if version != 1 { + return nil, fmt.Errorf("cannot handle snappy-java codec version other than 1 (got %d)", version) + } + // b[12:16] is the "compatible version"; ignore for now + var ( + decoded = make([]byte, 0, len(b)) + chunk []byte + err error + ) + for i := 16; i < len(b); { + n := int(binary.BigEndian.Uint32(b[i : i+4])) + i += 4 + chunk, err = snappy.Decode(chunk, b[i:i+n]) + if err != nil { + return nil, err + } + i += n + decoded = append(decoded, chunk...) + } + return decoded, nil +} diff --git a/docs/sink-configuration.md b/docs/sink-configuration.md index fd307f9d9d..5ea8a3b610 100644 --- a/docs/sink-configuration.md +++ b/docs/sink-configuration.md @@ -78,6 +78,25 @@ The following options are available: * `tenant` - Hawkular-Metrics tenantId (default: `heapster`) * `labelToTenant` - Hawkular-Metrics uses given label's value as tenant value when storing data +### Kafka +This sink supports monitoring metrics and events. +To use the kafka sink add the following flag: + +``` +--sink=kafka:[?] +``` +Normally, kafka server has multi brokers, so brokers' list need be configured for producer. +So, we can set `KAFKA_SERVER_URL` to a dummy value, and provide kafka brokers' list in url's query string. +Besides,the following options need be set in query string: + +* `timeseriestopic` - Kafka's topic for timeseries +* `eventstopic` - Kafka's topic for events + +Like this: +``` +--sink=kafka:http://kafka/?brokers=0.0.0.0:9092&brokers=0.0.0.0:9093×eriestopic=test&eventstopic=test +``` + ## Modifying the sinks at runtime Using the `/api/v1/sinks` endpoint, it is possible to fetch the sinks diff --git a/sinks/kafka/driver.go b/sinks/kafka/driver.go new file mode 100644 index 0000000000..328672e99a --- /dev/null +++ b/sinks/kafka/driver.go @@ -0,0 +1,159 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "encoding/json" + "fmt" + "net/url" + "time" + + "github.com/golang/glog" + "github.com/optiopay/kafka" + "github.com/optiopay/kafka/proto" + "k8s.io/heapster/extpoints" + sink_api "k8s.io/heapster/sinks/api" + kube_api "k8s.io/kubernetes/pkg/api" +) + +const ( + partition int32 = 0 + brokerClientID string = "kafka-sink" + brokerDialTimeout time.Duration = 10 * time.Second + brokerDialRetryLimit int = 10 + brokerDialRetryWait time.Duration = 500 * time.Millisecond + brokerAllowTopicCreation bool = true + brokerLeaderRetryLimit int = 10 + brokerLeaderRetryWait time.Duration = 500 * time.Millisecond +) + +type kafkaSink struct { + producer kafka.Producer + timeSeriesTopic string + eventsTopic string + sinkBrokerHosts []string +} + +// START: ExternalSink interface implementations + +func (self *kafkaSink) Register(mds []sink_api.MetricDescriptor) error { + return nil +} + +func (self *kafkaSink) Unregister(mds []sink_api.MetricDescriptor) error { + return nil +} + +func (self *kafkaSink) StoreTimeseries(timeseries []sink_api.Timeseries) error { + if timeseries == nil || len(timeseries) <= 0 { + return nil + } + for _, t := range timeseries { + err := self.produceKafkaMessage(t, self.timeSeriesTopic) + if err != nil { + return fmt.Errorf("failed to produce Kafka messages: %s", err) + } + } + return nil +} + +func (self *kafkaSink) StoreEvents(events []kube_api.Event) error { + if events == nil || len(events) <= 0 { + return nil + } + for _, event := range events { + err := self.produceKafkaMessage(event, self.eventsTopic) + if err != nil { + return fmt.Errorf("failed to produce Kafka messages: %s", err) + } + } + return nil +} + +// produceKafkaMessage produces messages to kafka +func (self *kafkaSink) produceKafkaMessage(v interface{}, topic string) error { + if v == nil { + return nil + } + jsonItems, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("failed to transform the items to json : %s", err) + } + message := &proto.Message{Value: []byte(string(jsonItems))} + _, err = self.producer.Produce(topic, partition, message) + if err != nil { + return fmt.Errorf("failed to produce message to %s:%d: %s", topic, partition, err) + } + return nil +} + +func (self *kafkaSink) DebugInfo() string { + info := fmt.Sprintf("%s\n", self.Name()) + info += fmt.Sprintf("There are two kafka's topics: %s,%s:\n", self.eventsTopic, self.timeSeriesTopic) + info += fmt.Sprintf("The kafka's broker list is: %s", self.sinkBrokerHosts) + return info +} + +func (self *kafkaSink) Name() string { + return "Apache-Kafka Sink" +} + +func init() { + extpoints.SinkFactories.Register(NewKafkaSink, "kafka") +} + +func NewKafkaSink(uri *url.URL, _ extpoints.HeapsterConf) ([]sink_api.ExternalSink, error) { + var kafkaSink kafkaSink + opts, _ := url.ParseQuery(uri.RawQuery) + + if len(opts["timeseriestopic"]) < 1 { + return nil, fmt.Errorf("There is no timeseriestopic assign for config kafka-sink") + } + kafkaSink.timeSeriesTopic = opts["eventstopic"][0] + + if len(opts["eventstopic"]) < 1 { + return nil, fmt.Errorf("There is no eventstopic assign for config kafka-sink") + } + kafkaSink.eventsTopic = opts["eventstopic"][0] + + if len(opts["brokers"]) < 1 { + return nil, fmt.Errorf("There is no broker assign for connecting kafka broker") + } + kafkaSink.sinkBrokerHosts = append(kafkaSink.sinkBrokerHosts, opts["brokers"]...) + + //connect to kafka cluster + brokerConf := kafka.NewBrokerConf(brokerClientID) + brokerConf.DialTimeout = brokerDialTimeout + brokerConf.DialRetryLimit = brokerDialRetryLimit + brokerConf.DialRetryWait = brokerDialRetryWait + brokerConf.LeaderRetryLimit = brokerLeaderRetryLimit + brokerConf.LeaderRetryWait = brokerLeaderRetryWait + brokerConf.AllowTopicCreation = true + + broker, err := kafka.Dial(kafkaSink.sinkBrokerHosts, brokerConf) + if err != nil { + glog.Errorf("connect to kafka cluster fail: %s", err) + return nil, fmt.Errorf("connect to kafka cluster fail: %s", err) + } + defer broker.Close() + + //create kafka producer + conf := kafka.NewProducerConf() + conf.RequiredAcks = proto.RequiredAcksLocal + sinkProducer := broker.Producer(conf) + kafkaSink.producer = sinkProducer + glog.Infof("created kafka sink successfully with brokers: %v", kafkaSink.sinkBrokerHosts) + return []sink_api.ExternalSink{&kafkaSink}, nil +} diff --git a/sinks/kafka/driver_test.go b/sinks/kafka/driver_test.go new file mode 100644 index 0000000000..b2d7d2ae04 --- /dev/null +++ b/sinks/kafka/driver_test.go @@ -0,0 +1,237 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "encoding/json" + "testing" + "time" + + "github.com/optiopay/kafka/proto" + "github.com/stretchr/testify/assert" + sink_api "k8s.io/heapster/sinks/api" + kube_api "k8s.io/kubernetes/pkg/api" + kube_api_unv "k8s.io/kubernetes/pkg/api/unversioned" +) + +type msgProducedToKafka struct { + message string +} + +type fakeKafkaProducer struct { + msgs []msgProducedToKafka +} + +type fakeKafkaSink struct { + sink_api.ExternalSink + fakeProducer *fakeKafkaProducer +} + +func NewFakeKafkaProducer() *fakeKafkaProducer { + return &fakeKafkaProducer{[]msgProducedToKafka{}} +} + +func (producer *fakeKafkaProducer) Produce(topic string, partition int32, messages ...*proto.Message) (offset int64, err error) { + for index := range messages { + producer.msgs = append(producer.msgs, msgProducedToKafka{string(messages[index].Value)}) + } + return 0, nil +} + +// Returns a fake kafka sink. +func NewFakeSink() fakeKafkaSink { + producer := NewFakeKafkaProducer() + fakeTimeSeriesTopic := "kafkaTime-test-topic" + fakeEventsTopic := "kafkaEvent-test-topic" + fakesinkBrokerHosts := make([]string, 2) + return fakeKafkaSink{ + &kafkaSink{ + producer, + fakeTimeSeriesTopic, + fakeEventsTopic, + fakesinkBrokerHosts, + }, + producer, + } +} + +func TestStoreEventsNilInput(t *testing.T) { + fakeSink := NewFakeSink() + err := fakeSink.StoreEvents(nil) + assert.NoError(t, err) + assert.Equal(t, 0, len(fakeSink.fakeProducer.msgs)) +} + +func TestStoreEventsEmptyInput(t *testing.T) { + fakeSink := NewFakeSink() + err := fakeSink.StoreEvents([]kube_api.Event{}) + assert.NoError(t, err) + assert.Equal(t, 0, len(fakeSink.fakeProducer.msgs)) +} + +func TestStoreEventsSingleEventInput(t *testing.T) { + fakeSink := NewFakeSink() + eventTime := kube_api_unv.Unix(12345, 0) + eventSourceHostname := "event1HostName" + eventReason := "event1" + events := []kube_api.Event{ + { + Reason: eventReason, + LastTimestamp: eventTime, + Source: kube_api.EventSource{ + Host: eventSourceHostname, + }, + }, + } + //expect msg string + eventsJson, _ := json.Marshal(events[0]) + err := fakeSink.StoreEvents(events) + assert.NoError(t, err) + assert.Equal(t, 1, len(fakeSink.fakeProducer.msgs)) + assert.Equal(t, eventsJson, fakeSink.fakeProducer.msgs[0].message) +} + +func TestStoreEventsMultipleEventsInput(t *testing.T) { + fakeSink := NewFakeSink() + event1Time := kube_api_unv.Unix(12345, 0) + event2Time := kube_api_unv.Unix(12366, 0) + event1SourceHostname := "event1HostName" + event2SourceHostname := "event2HostName" + event1Reason := "event1" + event2Reason := "event2" + events := []kube_api.Event{ + { + Reason: event1Reason, + LastTimestamp: event1Time, + Source: kube_api.EventSource{ + Host: event1SourceHostname, + }, + }, + { + Reason: event2Reason, + LastTimestamp: event2Time, + Source: kube_api.EventSource{ + Host: event2SourceHostname, + }, + }, + } + err := fakeSink.StoreEvents(events) + assert.NoError(t, err) + assert.Equal(t, 2, len(fakeSink.fakeProducer.msgs)) + events0Json, _ := json.Marshal(events[0]) + events1Json, _ := json.Marshal(events[1]) + assert.Equal(t, string(events0Json), fakeSink.fakeProducer.msgs[0].message) + assert.Equal(t, string(events1Json), fakeSink.fakeProducer.msgs[1].message) +} + +func TestStoreTimeseriesNilInput(t *testing.T) { + fakeSink := NewFakeSink() + err := fakeSink.StoreTimeseries(nil) + assert.NoError(t, err) + assert.Equal(t, 0, len(fakeSink.fakeProducer.msgs)) +} + +func TestStoreTimeseriesEmptyInput(t *testing.T) { + fakeSink := NewFakeSink() + err := fakeSink.StoreTimeseries([]sink_api.Timeseries{}) + assert.NoError(t, err) + assert.Equal(t, 0, len(fakeSink.fakeProducer.msgs)) +} + +func TestStoreTimeseriesSingleTimeserieInput(t *testing.T) { + fakeSink := NewFakeSink() + + smd := sink_api.MetricDescriptor{ + ValueType: sink_api.ValueInt64, + Type: sink_api.MetricCumulative, + } + + l := make(map[string]string) + l["test"] = "notvisible" + l[sink_api.LabelHostname.Key] = "localhost" + l[sink_api.LabelContainerName.Key] = "docker" + l[sink_api.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" + + p := sink_api.Point{ + Name: "test/metric/1", + Labels: l, + Start: time.Now(), + End: time.Now(), + Value: int64(123456), + } + + timeseries := []sink_api.Timeseries{ + { + MetricDescriptor: &smd, + Point: &p, + }, + } + + err := fakeSink.StoreTimeseries(timeseries) + assert.NoError(t, err) + assert.Equal(t, 1, len(fakeSink.fakeProducer.msgs)) + timeseries1Json, _ := json.Marshal(timeseries[0]) + assert.Equal(t, string(timeseries1Json), fakeSink.fakeProducer.msgs[0].message) +} + +func TestStoreTimeseriesMultipleTimeseriesInput(t *testing.T) { + fakeSink := NewFakeSink() + + smd := sink_api.MetricDescriptor{ + ValueType: sink_api.ValueInt64, + Type: sink_api.MetricCumulative, + } + + l := make(map[string]string) + l["test"] = "notvisible" + l[sink_api.LabelHostname.Key] = "localhost" + l[sink_api.LabelContainerName.Key] = "docker" + l[sink_api.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" + + p1 := sink_api.Point{ + Name: "test/metric/1", + Labels: l, + Start: time.Now(), + End: time.Now(), + Value: int64(123456), + } + + p2 := sink_api.Point{ + Name: "test/metric/1", + Labels: l, + Start: time.Now(), + End: time.Now(), + Value: int64(123456), + } + + timeseries := []sink_api.Timeseries{ + { + MetricDescriptor: &smd, + Point: &p1, + }, + { + MetricDescriptor: &smd, + Point: &p2, + }, + } + + err := fakeSink.StoreTimeseries(timeseries) + assert.NoError(t, err) + assert.Equal(t, 2, len(fakeSink.fakeProducer.msgs)) + timeseries0Json, _ := json.Marshal(timeseries[0]) + timeseries1Json, _ := json.Marshal(timeseries[1]) + assert.Equal(t, string(timeseries0Json), fakeSink.fakeProducer.msgs[0].message) + assert.Equal(t, string(timeseries1Json), fakeSink.fakeProducer.msgs[1].message) +} diff --git a/sinks/modules.go b/sinks/modules.go index 20bd9262b5..48b44fb34e 100644 --- a/sinks/modules.go +++ b/sinks/modules.go @@ -20,4 +20,5 @@ import ( _ "k8s.io/heapster/sinks/gcmautoscaling" _ "k8s.io/heapster/sinks/hawkular" _ "k8s.io/heapster/sinks/influxdb" + _ "k8s.io/heapster/sinks/kafka" ) From 423f3012aaa196c65f0c6a681eddfcee0dae6ce9 Mon Sep 17 00:00:00 2001 From: huangyuqi Date: Tue, 20 Oct 2015 20:44:19 +0000 Subject: [PATCH 2/4] fix according to comments of mvdan --- docs/sink-configuration.md | 10 ++++------ sinks/kafka/driver.go | 16 ++++++++-------- sinks/kafka/driver_test.go | 6 +++--- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/sink-configuration.md b/docs/sink-configuration.md index 5ea8a3b610..0bade44300 100644 --- a/docs/sink-configuration.md +++ b/docs/sink-configuration.md @@ -82,9 +82,8 @@ The following options are available: This sink supports monitoring metrics and events. To use the kafka sink add the following flag: -``` ---sink=kafka:[?] -``` + --sink=kafka:[?] + Normally, kafka server has multi brokers, so brokers' list need be configured for producer. So, we can set `KAFKA_SERVER_URL` to a dummy value, and provide kafka brokers' list in url's query string. Besides,the following options need be set in query string: @@ -93,9 +92,8 @@ Besides,the following options need be set in query string: * `eventstopic` - Kafka's topic for events Like this: -``` ---sink=kafka:http://kafka/?brokers=0.0.0.0:9092&brokers=0.0.0.0:9093×eriestopic=test&eventstopic=test -``` + + --sink=kafka:http://kafka/?brokers=0.0.0.0:9092&brokers=0.0.0.0:9093×eriestopic=test&eventstopic=test ## Modifying the sinks at runtime diff --git a/sinks/kafka/driver.go b/sinks/kafka/driver.go index 328672e99a..7c7697f74c 100644 --- a/sinks/kafka/driver.go +++ b/sinks/kafka/driver.go @@ -29,14 +29,14 @@ import ( ) const ( - partition int32 = 0 - brokerClientID string = "kafka-sink" - brokerDialTimeout time.Duration = 10 * time.Second - brokerDialRetryLimit int = 10 - brokerDialRetryWait time.Duration = 500 * time.Millisecond - brokerAllowTopicCreation bool = true - brokerLeaderRetryLimit int = 10 - brokerLeaderRetryWait time.Duration = 500 * time.Millisecond + partition = 0 + brokerClientID = "kafka-sink" + brokerDialTimeout = 10 * time.Second + brokerDialRetryLimit = 1 + brokerDialRetryWait = 500 * time.Millisecond + brokerAllowTopicCreation = true + brokerLeaderRetryLimit = 10 + brokerLeaderRetryWait = 500 * time.Millisecond ) type kafkaSink struct { diff --git a/sinks/kafka/driver_test.go b/sinks/kafka/driver_test.go index b2d7d2ae04..749c3e0aae 100644 --- a/sinks/kafka/driver_test.go +++ b/sinks/kafka/driver_test.go @@ -43,9 +43,9 @@ func NewFakeKafkaProducer() *fakeKafkaProducer { return &fakeKafkaProducer{[]msgProducedToKafka{}} } -func (producer *fakeKafkaProducer) Produce(topic string, partition int32, messages ...*proto.Message) (offset int64, err error) { - for index := range messages { - producer.msgs = append(producer.msgs, msgProducedToKafka{string(messages[index].Value)}) +func (producer *fakeKafkaProducer) Produce(topic string, partition int32, messages ...*proto.Message) (int64, error) { + for _, msg := range messages { + producer.msgs = append(producer.msgs, msgProducedToKafka{string(msg.Value)}) } return 0, nil } From f6348139ed34e7aa749abfeed1e78ff241911a83 Mon Sep 17 00:00:00 2001 From: huangyuqi Date: Wed, 21 Oct 2015 13:40:57 +0000 Subject: [PATCH 3/4] fix some ignored err & remove redundant test case --- sinks/kafka/driver.go | 10 ++++++---- sinks/kafka/driver_test.go | 34 +++++++++++++--------------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/sinks/kafka/driver.go b/sinks/kafka/driver.go index 7c7697f74c..d7a6391105 100644 --- a/sinks/kafka/driver.go +++ b/sinks/kafka/driver.go @@ -32,7 +32,7 @@ const ( partition = 0 brokerClientID = "kafka-sink" brokerDialTimeout = 10 * time.Second - brokerDialRetryLimit = 1 + brokerDialRetryLimit = 10 brokerDialRetryWait = 500 * time.Millisecond brokerAllowTopicCreation = true brokerLeaderRetryLimit = 10 @@ -116,7 +116,10 @@ func init() { func NewKafkaSink(uri *url.URL, _ extpoints.HeapsterConf) ([]sink_api.ExternalSink, error) { var kafkaSink kafkaSink - opts, _ := url.ParseQuery(uri.RawQuery) + opts, err := url.ParseQuery(uri.RawQuery) + if err != nil { + return nil, fmt.Errorf("failed to parser url's query string: %s", err) + } if len(opts["timeseriestopic"]) < 1 { return nil, fmt.Errorf("There is no timeseriestopic assign for config kafka-sink") @@ -144,8 +147,7 @@ func NewKafkaSink(uri *url.URL, _ extpoints.HeapsterConf) ([]sink_api.ExternalSi broker, err := kafka.Dial(kafkaSink.sinkBrokerHosts, brokerConf) if err != nil { - glog.Errorf("connect to kafka cluster fail: %s", err) - return nil, fmt.Errorf("connect to kafka cluster fail: %s", err) + return nil, fmt.Errorf("failed to connect to kafka cluster: %s", err) } defer broker.Close() diff --git a/sinks/kafka/driver_test.go b/sinks/kafka/driver_test.go index 749c3e0aae..4cc52ab7ec 100644 --- a/sinks/kafka/driver_test.go +++ b/sinks/kafka/driver_test.go @@ -67,13 +67,6 @@ func NewFakeSink() fakeKafkaSink { } } -func TestStoreEventsNilInput(t *testing.T) { - fakeSink := NewFakeSink() - err := fakeSink.StoreEvents(nil) - assert.NoError(t, err) - assert.Equal(t, 0, len(fakeSink.fakeProducer.msgs)) -} - func TestStoreEventsEmptyInput(t *testing.T) { fakeSink := NewFakeSink() err := fakeSink.StoreEvents([]kube_api.Event{}) @@ -96,8 +89,9 @@ func TestStoreEventsSingleEventInput(t *testing.T) { }, } //expect msg string - eventsJson, _ := json.Marshal(events[0]) - err := fakeSink.StoreEvents(events) + eventsJson, err := json.Marshal(events[0]) + assert.NoError(t, err) + err = fakeSink.StoreEvents(events) assert.NoError(t, err) assert.Equal(t, 1, len(fakeSink.fakeProducer.msgs)) assert.Equal(t, eventsJson, fakeSink.fakeProducer.msgs[0].message) @@ -130,19 +124,14 @@ func TestStoreEventsMultipleEventsInput(t *testing.T) { err := fakeSink.StoreEvents(events) assert.NoError(t, err) assert.Equal(t, 2, len(fakeSink.fakeProducer.msgs)) - events0Json, _ := json.Marshal(events[0]) - events1Json, _ := json.Marshal(events[1]) + events0Json, err := json.Marshal(events[0]) + assert.NoError(t, err) + events1Json, err := json.Marshal(events[1]) + assert.NoError(t, err) assert.Equal(t, string(events0Json), fakeSink.fakeProducer.msgs[0].message) assert.Equal(t, string(events1Json), fakeSink.fakeProducer.msgs[1].message) } -func TestStoreTimeseriesNilInput(t *testing.T) { - fakeSink := NewFakeSink() - err := fakeSink.StoreTimeseries(nil) - assert.NoError(t, err) - assert.Equal(t, 0, len(fakeSink.fakeProducer.msgs)) -} - func TestStoreTimeseriesEmptyInput(t *testing.T) { fakeSink := NewFakeSink() err := fakeSink.StoreTimeseries([]sink_api.Timeseries{}) @@ -182,7 +171,8 @@ func TestStoreTimeseriesSingleTimeserieInput(t *testing.T) { err := fakeSink.StoreTimeseries(timeseries) assert.NoError(t, err) assert.Equal(t, 1, len(fakeSink.fakeProducer.msgs)) - timeseries1Json, _ := json.Marshal(timeseries[0]) + timeseries1Json, err := json.Marshal(timeseries[0]) + assert.NoError(t, err) assert.Equal(t, string(timeseries1Json), fakeSink.fakeProducer.msgs[0].message) } @@ -230,8 +220,10 @@ func TestStoreTimeseriesMultipleTimeseriesInput(t *testing.T) { err := fakeSink.StoreTimeseries(timeseries) assert.NoError(t, err) assert.Equal(t, 2, len(fakeSink.fakeProducer.msgs)) - timeseries0Json, _ := json.Marshal(timeseries[0]) - timeseries1Json, _ := json.Marshal(timeseries[1]) + timeseries0Json, err := json.Marshal(timeseries[0]) + assert.NoError(t, err) + timeseries1Json, err := json.Marshal(timeseries[1]) + assert.NoError(t, err) assert.Equal(t, string(timeseries0Json), fakeSink.fakeProducer.msgs[0].message) assert.Equal(t, string(timeseries1Json), fakeSink.fakeProducer.msgs[1].message) } From 4bbc88a8043914c427ba118cddface04f7173b22 Mon Sep 17 00:00:00 2001 From: huangyuqi Date: Thu, 29 Oct 2015 16:29:33 +0800 Subject: [PATCH 4/4] modify the data struct of event&metric --- sinks/kafka/driver.go | 57 +++++++++++++++++++++++++---- sinks/kafka/driver_test.go | 73 +++++++++++++++++++++++--------------- 2 files changed, 95 insertions(+), 35 deletions(-) diff --git a/sinks/kafka/driver.go b/sinks/kafka/driver.go index d7a6391105..66884d9bf6 100644 --- a/sinks/kafka/driver.go +++ b/sinks/kafka/driver.go @@ -32,11 +32,11 @@ const ( partition = 0 brokerClientID = "kafka-sink" brokerDialTimeout = 10 * time.Second - brokerDialRetryLimit = 10 - brokerDialRetryWait = 500 * time.Millisecond + brokerDialRetryLimit = 1 + brokerDialRetryWait = 0 brokerAllowTopicCreation = true - brokerLeaderRetryLimit = 10 - brokerLeaderRetryWait = 500 * time.Millisecond + brokerLeaderRetryLimit = 1 + brokerLeaderRetryWait = 0 ) type kafkaSink struct { @@ -46,6 +46,22 @@ type kafkaSink struct { sinkBrokerHosts []string } +type kafkaSinkPoint struct { + MetricsName string + MetricsValue interface{} + MetricsTimestamp time.Time + MetricsTags map[string]string +} + +type kafkaSinkEvent struct { + EventMessage string + EventReason string + EventTimestamp time.Time + EventCount int + EventInvolvedObject interface{} + EventSource interface{} +} + // START: ExternalSink interface implementations func (self *kafkaSink) Register(mds []sink_api.MetricDescriptor) error { @@ -61,7 +77,25 @@ func (self *kafkaSink) StoreTimeseries(timeseries []sink_api.Timeseries) error { return nil } for _, t := range timeseries { - err := self.produceKafkaMessage(t, self.timeSeriesTopic) + seriesName := t.Point.Name + if t.MetricDescriptor.Units.String() != "" { + seriesName = fmt.Sprintf("%s_%s", seriesName, t.MetricDescriptor.Units.String()) + } + if t.MetricDescriptor.Type.String() != "" { + seriesName = fmt.Sprintf("%s_%s", seriesName, t.MetricDescriptor.Type.String()) + } + sinkPoint := kafkaSinkPoint{ + MetricsName: seriesName, + MetricsValue: t.Point.Value, + MetricsTimestamp: t.Point.End.UTC(), + MetricsTags: make(map[string]string, len(t.Point.Labels)), + } + for key, value := range t.Point.Labels { + if value != "" { + sinkPoint.MetricsTags[key] = value + } + } + err := self.produceKafkaMessage(sinkPoint, self.timeSeriesTopic) if err != nil { return fmt.Errorf("failed to produce Kafka messages: %s", err) } @@ -74,7 +108,16 @@ func (self *kafkaSink) StoreEvents(events []kube_api.Event) error { return nil } for _, event := range events { - err := self.produceKafkaMessage(event, self.eventsTopic) + sinkEvent := kafkaSinkEvent{ + EventMessage: event.Message, + EventReason: event.Reason, + EventTimestamp: event.LastTimestamp.UTC(), + EventCount: event.Count, + EventInvolvedObject: event.InvolvedObject, + EventSource: event.Source, + } + + err := self.produceKafkaMessage(sinkEvent, self.eventsTopic) if err != nil { return fmt.Errorf("failed to produce Kafka messages: %s", err) } @@ -124,7 +167,7 @@ func NewKafkaSink(uri *url.URL, _ extpoints.HeapsterConf) ([]sink_api.ExternalSi if len(opts["timeseriestopic"]) < 1 { return nil, fmt.Errorf("There is no timeseriestopic assign for config kafka-sink") } - kafkaSink.timeSeriesTopic = opts["eventstopic"][0] + kafkaSink.timeSeriesTopic = opts["timeseriestopic"][0] if len(opts["eventstopic"]) < 1 { return nil, fmt.Errorf("There is no eventstopic assign for config kafka-sink") diff --git a/sinks/kafka/driver_test.go b/sinks/kafka/driver_test.go index 4cc52ab7ec..280a902c2f 100644 --- a/sinks/kafka/driver_test.go +++ b/sinks/kafka/driver_test.go @@ -15,10 +15,11 @@ package kafka import ( - "encoding/json" + _ "encoding/json" "testing" "time" + "fmt" "github.com/optiopay/kafka/proto" "github.com/stretchr/testify/assert" sink_api "k8s.io/heapster/sinks/api" @@ -89,33 +90,35 @@ func TestStoreEventsSingleEventInput(t *testing.T) { }, } //expect msg string - eventsJson, err := json.Marshal(events[0]) + timeStr, err := eventTime.MarshalJSON() assert.NoError(t, err) + + msgString := fmt.Sprintf(`{"EventMessage":"","EventReason":"%s","EventTimestamp":%s,"EventCount":0,"EventInvolvedObject":{},"EventSource":{"host":"%s"}}`, eventReason, string(timeStr), eventSourceHostname) err = fakeSink.StoreEvents(events) assert.NoError(t, err) + assert.Equal(t, 1, len(fakeSink.fakeProducer.msgs)) - assert.Equal(t, eventsJson, fakeSink.fakeProducer.msgs[0].message) + assert.Equal(t, msgString, fakeSink.fakeProducer.msgs[0].message) } func TestStoreEventsMultipleEventsInput(t *testing.T) { fakeSink := NewFakeSink() - event1Time := kube_api_unv.Unix(12345, 0) - event2Time := kube_api_unv.Unix(12366, 0) + eventTime := kube_api_unv.Unix(12345, 0) event1SourceHostname := "event1HostName" event2SourceHostname := "event2HostName" - event1Reason := "event1" - event2Reason := "event2" + event1Reason := "eventReason1" + event2Reason := "eventReason2" events := []kube_api.Event{ { Reason: event1Reason, - LastTimestamp: event1Time, + LastTimestamp: eventTime, Source: kube_api.EventSource{ Host: event1SourceHostname, }, }, { Reason: event2Reason, - LastTimestamp: event2Time, + LastTimestamp: eventTime, Source: kube_api.EventSource{ Host: event2SourceHostname, }, @@ -124,12 +127,15 @@ func TestStoreEventsMultipleEventsInput(t *testing.T) { err := fakeSink.StoreEvents(events) assert.NoError(t, err) assert.Equal(t, 2, len(fakeSink.fakeProducer.msgs)) - events0Json, err := json.Marshal(events[0]) - assert.NoError(t, err) - events1Json, err := json.Marshal(events[1]) + + timeStr, err := eventTime.MarshalJSON() assert.NoError(t, err) - assert.Equal(t, string(events0Json), fakeSink.fakeProducer.msgs[0].message) - assert.Equal(t, string(events1Json), fakeSink.fakeProducer.msgs[1].message) + + msgString1 := fmt.Sprintf(`{"EventMessage":"","EventReason":"%s","EventTimestamp":%s,"EventCount":0,"EventInvolvedObject":{},"EventSource":{"host":"%s"}}`, event1Reason, string(timeStr), event1SourceHostname) + assert.Equal(t, msgString1, fakeSink.fakeProducer.msgs[0].message) + + msgString2 := fmt.Sprintf(`{"EventMessage":"","EventReason":"%s","EventTimestamp":%s,"EventCount":0,"EventInvolvedObject":{},"EventSource":{"host":"%s"}}`, event2Reason, string(timeStr), event2SourceHostname) + assert.Equal(t, msgString2, fakeSink.fakeProducer.msgs[1].message) } func TestStoreTimeseriesEmptyInput(t *testing.T) { @@ -152,12 +158,13 @@ func TestStoreTimeseriesSingleTimeserieInput(t *testing.T) { l[sink_api.LabelHostname.Key] = "localhost" l[sink_api.LabelContainerName.Key] = "docker" l[sink_api.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" + timeNow := time.Now() p := sink_api.Point{ Name: "test/metric/1", Labels: l, - Start: time.Now(), - End: time.Now(), + Start: timeNow, + End: timeNow, Value: int64(123456), } @@ -170,10 +177,15 @@ func TestStoreTimeseriesSingleTimeserieInput(t *testing.T) { err := fakeSink.StoreTimeseries(timeseries) assert.NoError(t, err) + assert.Equal(t, 1, len(fakeSink.fakeProducer.msgs)) - timeseries1Json, err := json.Marshal(timeseries[0]) + + timeStr, err := timeNow.MarshalJSON() assert.NoError(t, err) - assert.Equal(t, string(timeseries1Json), fakeSink.fakeProducer.msgs[0].message) + + msgString := fmt.Sprintf(`{"MetricsName":"test/metric/1_cumulative","MetricsValue":123456,"MetricsTimestamp":%s,"MetricsTags":{"container_name":"docker","hostname":"localhost","pod_id":"aaaa-bbbb-cccc-dddd","test":"notvisible"}}`, timeStr) + + assert.Equal(t, msgString, fakeSink.fakeProducer.msgs[0].message) } func TestStoreTimeseriesMultipleTimeseriesInput(t *testing.T) { @@ -189,21 +201,22 @@ func TestStoreTimeseriesMultipleTimeseriesInput(t *testing.T) { l[sink_api.LabelHostname.Key] = "localhost" l[sink_api.LabelContainerName.Key] = "docker" l[sink_api.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd" + timeNow := time.Now() p1 := sink_api.Point{ Name: "test/metric/1", Labels: l, - Start: time.Now(), - End: time.Now(), + Start: timeNow, + End: timeNow, Value: int64(123456), } p2 := sink_api.Point{ Name: "test/metric/1", Labels: l, - Start: time.Now(), - End: time.Now(), - Value: int64(123456), + Start: timeNow, + End: timeNow, + Value: int64(654321), } timeseries := []sink_api.Timeseries{ @@ -219,11 +232,15 @@ func TestStoreTimeseriesMultipleTimeseriesInput(t *testing.T) { err := fakeSink.StoreTimeseries(timeseries) assert.NoError(t, err) + assert.Equal(t, 2, len(fakeSink.fakeProducer.msgs)) - timeseries0Json, err := json.Marshal(timeseries[0]) - assert.NoError(t, err) - timeseries1Json, err := json.Marshal(timeseries[1]) + + timeStr, err := timeNow.MarshalJSON() assert.NoError(t, err) - assert.Equal(t, string(timeseries0Json), fakeSink.fakeProducer.msgs[0].message) - assert.Equal(t, string(timeseries1Json), fakeSink.fakeProducer.msgs[1].message) + + msgString1 := fmt.Sprintf(`{"MetricsName":"test/metric/1_cumulative","MetricsValue":123456,"MetricsTimestamp":%s,"MetricsTags":{"container_name":"docker","hostname":"localhost","pod_id":"aaaa-bbbb-cccc-dddd","test":"notvisible"}}`, timeStr) + assert.Equal(t, msgString1, fakeSink.fakeProducer.msgs[0].message) + + msgString2 := fmt.Sprintf(`{"MetricsName":"test/metric/1_cumulative","MetricsValue":654321,"MetricsTimestamp":%s,"MetricsTags":{"container_name":"docker","hostname":"localhost","pod_id":"aaaa-bbbb-cccc-dddd","test":"notvisible"}}`, timeStr) + assert.Equal(t, msgString2, fakeSink.fakeProducer.msgs[1].message) }