``` ```
import org.apache.kafka.common.security.scram.internals.ScramFormatter;
import org.apache.kafka.common.security.scram.internals.ScramMechanism;
import java.nio.charset.StandardCharsets;
public class App {
public static String bytesToHex(byte[] in) {
final StringBuilder builder = new StringBuilder();
for(byte b : in) {
builder.append(String.format("0x%02x, ", b));
}
return builder.toString();
}
public static void main(String[] args) throws NoSuchAlgorithmException, InvalidKeyException {
int digestIterations = 4096;
String password = "hello";
byte[] salt = "world".getBytes(StandardCharsets.UTF_8);
byte[] saltedPassword = new ScramFormatter(ScramMechanism.SCRAM_SHA_256)
.saltedPassword(password, salt, digestIterations);
System.out.println(bytesToHex(saltedPassword));
}
}
*/
func TestScramSaltedPasswordSha512(t *testing.T) {
password := []byte("hello")
salt := []byte("world")
formatter := scramFormatter{mechanism: SCRAM_MECHANISM_SHA_512}
result, _ := formatter.saltedPassword(password, salt, 4096)
// calculated using ScramFormatter (see comment above)
expected := []byte{
0x35, 0x0c, 0x77, 0x84, 0x8a, 0x63, 0x06, 0x92, 0x00,
0x6e, 0xc6, 0x6a, 0x0c, 0x39, 0xeb, 0xb0, 0x00, 0xd3,
0xf8, 0x8a, 0x94, 0xae, 0x7f, 0x8c, 0xcd, 0x1d, 0x92,
0x52, 0x6c, 0x5b, 0x16, 0x15, 0x86, 0x3b, 0xde, 0xa1,
0x6c, 0x12, 0x9a, 0x7b, 0x09, 0xed, 0x0e, 0x38, 0xf2,
0x07, 0x4d, 0x2f, 0xe2, 0x9f, 0x0f, 0x41, 0xe1, 0xfb,
0x00, 0xc1, 0xd3, 0xbd, 0xd3, 0xfd, 0x51, 0x0b, 0xa9,
0x8f,
}
if !bytes.Equal(result, expected) {
t.Errorf("saltedPassword SHA-512 failed, expected: %v, result: %v", expected, result)
}
}
func TestScramSaltedPasswordSha256(t *testing.T) {
password := []byte("hello")
salt := []byte("world")
formatter := scramFormatter{mechanism: SCRAM_MECHANISM_SHA_256}
result, _ := formatter.saltedPassword(password, salt, 4096)
// calculated using ScramFormatter (see comment above)
expected := []byte{
0xc1, 0x55, 0x53, 0x03, 0xda, 0x30, 0x9f, 0x6b, 0x7d,
0x1e, 0x8f, 0xe4, 0x56, 0x36, 0xbf, 0xdd, 0xdc, 0x4b,
0xf5, 0x64, 0x05, 0xe7, 0xe9, 0x4e, 0x9d, 0x15, 0xf0,
0xe7, 0xb9, 0xcb, 0xd3, 0x80,
}
if !bytes.Equal(result, expected) {
t.Errorf("saltedPassword SHA-256 failed, expected: %v, result: %v", expected, result)
}
}
golang-github-ibm-sarama-1.43.2/sticky_assignor_user_data.go 0000664 0000000 0000000 00000006350 14612567413 0024145 0 ustar 00root root 0000000 0000000 package sarama
type topicPartitionAssignment struct {
Topic string
Partition int32
}
type StickyAssignorUserData interface {
partitions() []topicPartitionAssignment
hasGeneration() bool
generation() int
}
// StickyAssignorUserDataV0 holds topic partition information for an assignment
type StickyAssignorUserDataV0 struct {
Topics map[string][]int32
topicPartitions []topicPartitionAssignment
}
func (m *StickyAssignorUserDataV0) encode(pe packetEncoder) error {
if err := pe.putArrayLength(len(m.Topics)); err != nil {
return err
}
for topic, partitions := range m.Topics {
if err := pe.putString(topic); err != nil {
return err
}
if err := pe.putInt32Array(partitions); err != nil {
return err
}
}
return nil
}
func (m *StickyAssignorUserDataV0) decode(pd packetDecoder) (err error) {
var topicLen int
if topicLen, err = pd.getArrayLength(); err != nil {
return
}
m.Topics = make(map[string][]int32, topicLen)
for i := 0; i < topicLen; i++ {
var topic string
if topic, err = pd.getString(); err != nil {
return
}
if m.Topics[topic], err = pd.getInt32Array(); err != nil {
return
}
}
m.topicPartitions = populateTopicPartitions(m.Topics)
return nil
}
func (m *StickyAssignorUserDataV0) partitions() []topicPartitionAssignment { return m.topicPartitions }
func (m *StickyAssignorUserDataV0) hasGeneration() bool { return false }
func (m *StickyAssignorUserDataV0) generation() int { return defaultGeneration }
// StickyAssignorUserDataV1 holds topic partition information for an assignment
type StickyAssignorUserDataV1 struct {
Topics map[string][]int32
Generation int32
topicPartitions []topicPartitionAssignment
}
func (m *StickyAssignorUserDataV1) encode(pe packetEncoder) error {
if err := pe.putArrayLength(len(m.Topics)); err != nil {
return err
}
for topic, partitions := range m.Topics {
if err := pe.putString(topic); err != nil {
return err
}
if err := pe.putInt32Array(partitions); err != nil {
return err
}
}
pe.putInt32(m.Generation)
return nil
}
func (m *StickyAssignorUserDataV1) decode(pd packetDecoder) (err error) {
var topicLen int
if topicLen, err = pd.getArrayLength(); err != nil {
return
}
m.Topics = make(map[string][]int32, topicLen)
for i := 0; i < topicLen; i++ {
var topic string
if topic, err = pd.getString(); err != nil {
return
}
if m.Topics[topic], err = pd.getInt32Array(); err != nil {
return
}
}
m.Generation, err = pd.getInt32()
if err != nil {
return err
}
m.topicPartitions = populateTopicPartitions(m.Topics)
return nil
}
func (m *StickyAssignorUserDataV1) partitions() []topicPartitionAssignment { return m.topicPartitions }
func (m *StickyAssignorUserDataV1) hasGeneration() bool { return true }
func (m *StickyAssignorUserDataV1) generation() int { return int(m.Generation) }
func populateTopicPartitions(topics map[string][]int32) []topicPartitionAssignment {
topicPartitions := make([]topicPartitionAssignment, 0)
for topic, partitions := range topics {
for _, partition := range partitions {
topicPartitions = append(topicPartitions, topicPartitionAssignment{Topic: topic, Partition: partition})
}
}
return topicPartitions
}
golang-github-ibm-sarama-1.43.2/sticky_assignor_user_data_test.go 0000664 0000000 0000000 00000003357 14612567413 0025210 0 ustar 00root root 0000000 0000000 package sarama
import (
"encoding/base64"
"testing"
)
func TestStickyAssignorUserDataV0(t *testing.T) {
// Single topic with deterministic ordering across encode-decode
req := &StickyAssignorUserDataV0{}
data := decodeUserDataBytes(t, "AAAAAQADdDAzAAAAAQAAAAU=")
testDecodable(t, "", req, data)
testEncodable(t, "", req, data)
// Multiple partitions
req = &StickyAssignorUserDataV0{}
data = decodeUserDataBytes(t, "AAAAAQADdDE4AAAAEgAAAAAAAAABAAAAAgAAAAMAAAAEAAAABQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAAAAwAAAANAAAADgAAAA8AAAAQAAAAEQ==")
testDecodable(t, "", req, data)
// Multiple topics and partitions
req = &StickyAssignorUserDataV0{}
data = decodeUserDataBytes(t, "AAAABQADdDEyAAAAAgAAAAIAAAAKAAN0MTEAAAABAAAABAADdDE0AAAAAQAAAAgAA3QxMwAAAAEAAAANAAN0MDkAAAABAAAABQ==")
testDecodable(t, "", req, data)
}
func TestStickyAssignorUserDataV1(t *testing.T) {
// Single topic with deterministic ordering across encode-decode
req := &StickyAssignorUserDataV1{}
data := decodeUserDataBytes(t, "AAAAAQADdDA2AAAAAgAAAAAAAAAE/////w==")
testDecodable(t, "", req, data)
testEncodable(t, "", req, data)
// Multiple topics and partitions
req = &StickyAssignorUserDataV1{}
data = decodeUserDataBytes(t, "AAAABgADdDEwAAAAAgAAAAIAAAAJAAN0MTIAAAACAAAAAwAAAAsAA3QxNAAAAAEAAAAEAAN0MTMAAAABAAAACwADdDE1AAAAAQAAAAwAA3QwOQAAAAEAAAAG/////w==")
testDecodable(t, "", req, data)
// Generation is populated
req = &StickyAssignorUserDataV1{}
data = decodeUserDataBytes(t, "AAAAAQAHdG9waWMwMQAAAAMAAAAAAAAAAQAAAAIAAAAB")
testDecodable(t, "", req, data)
}
func decodeUserDataBytes(t *testing.T, base64Data string) []byte {
data, err := base64.StdEncoding.DecodeString(base64Data)
if err != nil {
t.Errorf("Error decoding data: %v", err)
t.FailNow()
}
return data
}
golang-github-ibm-sarama-1.43.2/sync_group_request.go 0000664 0000000 0000000 00000007010 14612567413 0022635 0 ustar 00root root 0000000 0000000 package sarama
type SyncGroupRequestAssignment struct {
// MemberId contains the ID of the member to assign.
MemberId string
// Assignment contains the member assignment.
Assignment []byte
}
func (a *SyncGroupRequestAssignment) encode(pe packetEncoder, version int16) (err error) {
if err := pe.putString(a.MemberId); err != nil {
return err
}
if err := pe.putBytes(a.Assignment); err != nil {
return err
}
return nil
}
func (a *SyncGroupRequestAssignment) decode(pd packetDecoder, version int16) (err error) {
if a.MemberId, err = pd.getString(); err != nil {
return err
}
if a.Assignment, err = pd.getBytes(); err != nil {
return err
}
return nil
}
type SyncGroupRequest struct {
// Version defines the protocol version to use for encode and decode
Version int16
// GroupId contains the unique group identifier.
GroupId string
// GenerationId contains the generation of the group.
GenerationId int32
// MemberId contains the member ID assigned by the group.
MemberId string
// GroupInstanceId contains the unique identifier of the consumer instance provided by end user.
GroupInstanceId *string
// GroupAssignments contains each assignment.
GroupAssignments []SyncGroupRequestAssignment
}
func (s *SyncGroupRequest) encode(pe packetEncoder) (err error) {
if err := pe.putString(s.GroupId); err != nil {
return err
}
pe.putInt32(s.GenerationId)
if err := pe.putString(s.MemberId); err != nil {
return err
}
if s.Version >= 3 {
if err := pe.putNullableString(s.GroupInstanceId); err != nil {
return err
}
}
if err := pe.putArrayLength(len(s.GroupAssignments)); err != nil {
return err
}
for _, block := range s.GroupAssignments {
if err := block.encode(pe, s.Version); err != nil {
return err
}
}
return nil
}
func (s *SyncGroupRequest) decode(pd packetDecoder, version int16) (err error) {
s.Version = version
if s.GroupId, err = pd.getString(); err != nil {
return err
}
if s.GenerationId, err = pd.getInt32(); err != nil {
return err
}
if s.MemberId, err = pd.getString(); err != nil {
return err
}
if s.Version >= 3 {
if s.GroupInstanceId, err = pd.getNullableString(); err != nil {
return err
}
}
if numAssignments, err := pd.getArrayLength(); err != nil {
return err
} else if numAssignments > 0 {
s.GroupAssignments = make([]SyncGroupRequestAssignment, numAssignments)
for i := 0; i < numAssignments; i++ {
var block SyncGroupRequestAssignment
if err := block.decode(pd, s.Version); err != nil {
return err
}
s.GroupAssignments[i] = block
}
}
return nil
}
func (r *SyncGroupRequest) key() int16 {
return 14
}
func (r *SyncGroupRequest) version() int16 {
return r.Version
}
func (r *SyncGroupRequest) headerVersion() int16 {
return 1
}
func (r *SyncGroupRequest) isValidVersion() bool {
return r.Version >= 0 && r.Version <= 3
}
func (r *SyncGroupRequest) requiredVersion() KafkaVersion {
switch r.Version {
case 3:
return V2_3_0_0
case 2:
return V2_0_0_0
case 1:
return V0_11_0_0
case 0:
return V0_9_0_0
default:
return V2_3_0_0
}
}
func (r *SyncGroupRequest) AddGroupAssignment(memberId string, memberAssignment []byte) {
r.GroupAssignments = append(r.GroupAssignments, SyncGroupRequestAssignment{
MemberId: memberId,
Assignment: memberAssignment,
})
}
func (r *SyncGroupRequest) AddGroupAssignmentMember(
memberId string,
memberAssignment *ConsumerGroupMemberAssignment,
) error {
bin, err := encode(memberAssignment, nil)
if err != nil {
return err
}
r.AddGroupAssignment(memberId, bin)
return nil
}
golang-github-ibm-sarama-1.43.2/sync_group_request_test.go 0000664 0000000 0000000 00000004265 14612567413 0023705 0 ustar 00root root 0000000 0000000 package sarama
import (
"reflect"
"testing"
)
var (
emptySyncGroupRequest = []byte{
0, 3, 'f', 'o', 'o', // Group ID
0x00, 0x01, 0x02, 0x03, // Generation ID
0, 3, 'b', 'a', 'z', // Member ID
0, 0, 0, 0, // no assignments
}
populatedSyncGroupRequest = []byte{
0, 3, 'f', 'o', 'o', // Group ID
0x00, 0x01, 0x02, 0x03, // Generation ID
0, 3, 'b', 'a', 'z', // Member ID
0, 0, 0, 1, // one assignment
0, 3, 'b', 'a', 'z', // Member ID
0, 0, 0, 3, 'f', 'o', 'o', // Member assignment
}
)
func TestSyncGroupRequest(t *testing.T) {
var request *SyncGroupRequest
request = new(SyncGroupRequest)
request.GroupId = "foo"
request.GenerationId = 66051
request.MemberId = "baz"
testRequest(t, "empty", request, emptySyncGroupRequest)
request = new(SyncGroupRequest)
request.GroupId = "foo"
request.GenerationId = 66051
request.MemberId = "baz"
request.AddGroupAssignment("baz", []byte("foo"))
testRequest(t, "populated", request, populatedSyncGroupRequest)
}
var (
populatedSyncGroupRequestV3 = []byte{
0, 3, 'f', 'o', 'o', // Group ID
0x00, 0x01, 0x02, 0x03, // Generation ID
0, 3, 'b', 'a', 'z', // Member ID
0, 3, 'g', 'i', 'd', // GroupInstance ID
0, 0, 0, 1, // one assignment
0, 3, 'b', 'a', 'z', // Member ID
0, 0, 0, 3, 'f', 'o', 'o', // Member assignment
}
)
func TestSyncGroupRequestV3AndPlus(t *testing.T) {
groupInstanceId := "gid"
tests := []struct {
CaseName string
Version int16
MessageBytes []byte
Message *SyncGroupRequest
}{
{
"v3",
3,
populatedSyncGroupRequestV3,
&SyncGroupRequest{
Version: 3,
GroupId: "foo",
GenerationId: 0x00010203,
MemberId: "baz",
GroupInstanceId: &groupInstanceId,
GroupAssignments: []SyncGroupRequestAssignment{
{
MemberId: "baz",
Assignment: []byte("foo"),
},
},
},
},
}
for _, c := range tests {
request := new(SyncGroupRequest)
testVersionDecodable(t, c.CaseName, request, c.MessageBytes, c.Version)
if !reflect.DeepEqual(c.Message, request) {
t.Errorf("case %s decode failed, expected:%+v got %+v", c.CaseName, c.Message, request)
}
testEncodable(t, c.CaseName, c.Message, c.MessageBytes)
}
}
golang-github-ibm-sarama-1.43.2/sync_group_response.go 0000664 0000000 0000000 00000003512 14612567413 0023006 0 ustar 00root root 0000000 0000000 package sarama
import "time"
type SyncGroupResponse struct {
// Version defines the protocol version to use for encode and decode
Version int16
// ThrottleTime contains the duration in milliseconds for which the
// request was throttled due to a quota violation, or zero if the request
// did not violate any quota.
ThrottleTime int32
// Err contains the error code, or 0 if there was no error.
Err KError
// MemberAssignment contains the member assignment.
MemberAssignment []byte
}
func (r *SyncGroupResponse) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) {
assignment := new(ConsumerGroupMemberAssignment)
err := decode(r.MemberAssignment, assignment, nil)
return assignment, err
}
func (r *SyncGroupResponse) encode(pe packetEncoder) error {
if r.Version >= 1 {
pe.putInt32(r.ThrottleTime)
}
pe.putInt16(int16(r.Err))
return pe.putBytes(r.MemberAssignment)
}
func (r *SyncGroupResponse) decode(pd packetDecoder, version int16) (err error) {
r.Version = version
if r.Version >= 1 {
if r.ThrottleTime, err = pd.getInt32(); err != nil {
return err
}
}
kerr, err := pd.getInt16()
if err != nil {
return err
}
r.Err = KError(kerr)
r.MemberAssignment, err = pd.getBytes()
return
}
func (r *SyncGroupResponse) key() int16 {
return 14
}
func (r *SyncGroupResponse) version() int16 {
return r.Version
}
func (r *SyncGroupResponse) headerVersion() int16 {
return 0
}
func (r *SyncGroupResponse) isValidVersion() bool {
return r.Version >= 0 && r.Version <= 3
}
func (r *SyncGroupResponse) requiredVersion() KafkaVersion {
switch r.Version {
case 3:
return V2_3_0_0
case 2:
return V2_0_0_0
case 1:
return V0_11_0_0
case 0:
return V0_9_0_0
default:
return V2_3_0_0
}
}
func (r *SyncGroupResponse) throttleTime() time.Duration {
return time.Duration(r.ThrottleTime) * time.Millisecond
}
golang-github-ibm-sarama-1.43.2/sync_group_response_test.go 0000664 0000000 0000000 00000003051 14612567413 0024043 0 ustar 00root root 0000000 0000000 package sarama
import (
"reflect"
"testing"
)
var (
syncGroupResponseV0NoError = []byte{
0x00, 0x00, // No error
0, 0, 0, 3, 0x01, 0x02, 0x03, // Member assignment data
}
syncGroupResponseV0WithError = []byte{
0, 27, // ErrRebalanceInProgress
0, 0, 0, 0, // No member assignment data
}
syncGroupResponseV1NoError = []byte{
0, 0, 0, 100, // ThrottleTimeMs
0x00, 0x00, // No error
0, 0, 0, 3, 0x01, 0x02, 0x03, // Member assignment data
}
)
func TestSyncGroupResponse(t *testing.T) {
tests := []struct {
CaseName string
Version int16
MessageBytes []byte
Message *SyncGroupResponse
}{
{
"v0-noErr",
0,
syncGroupResponseV0NoError,
&SyncGroupResponse{
Version: 0,
Err: ErrNoError,
MemberAssignment: []byte{1, 2, 3},
},
},
{
"v0-Err",
0,
syncGroupResponseV0WithError,
&SyncGroupResponse{
Version: 0,
Err: ErrRebalanceInProgress,
MemberAssignment: []byte{},
},
},
{
"v1-noErr",
1,
syncGroupResponseV1NoError,
&SyncGroupResponse{
ThrottleTime: 100,
Version: 1,
Err: ErrNoError,
MemberAssignment: []byte{1, 2, 3},
},
},
}
for _, c := range tests {
response := new(SyncGroupResponse)
testVersionDecodable(t, c.CaseName, response, c.MessageBytes, c.Version)
if !reflect.DeepEqual(c.Message, response) {
t.Errorf("case %s decode failed, expected:%+v got %+v", c.CaseName, c.Message, response)
}
testEncodable(t, c.CaseName, c.Message, c.MessageBytes)
}
}
golang-github-ibm-sarama-1.43.2/sync_producer.go 0000664 0000000 0000000 00000013713 14612567413 0021563 0 ustar 00root root 0000000 0000000 package sarama
import "sync"
// SyncProducer publishes Kafka messages, blocking until they have been acknowledged. It routes messages to the correct
// broker, refreshing metadata as appropriate, and parses responses for errors. You must call Close() on a producer
// to avoid leaks, it may not be garbage-collected automatically when it passes out of scope.
//
// The SyncProducer comes with two caveats: it will generally be less efficient than the AsyncProducer, and the actual
// durability guarantee provided when a message is acknowledged depend on the configured value of `Producer.RequiredAcks`.
// There are configurations where a message acknowledged by the SyncProducer can still sometimes be lost.
//
// For implementation reasons, the SyncProducer requires `Producer.Return.Errors` and `Producer.Return.Successes` to
// be set to true in its configuration.
type SyncProducer interface {
// SendMessage produces a given message, and returns only when it either has
// succeeded or failed to produce. It will return the partition and the offset
// of the produced message, or an error if the message failed to produce.
SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error)
// SendMessages produces a given set of messages, and returns only when all
// messages in the set have either succeeded or failed. Note that messages
// can succeed and fail individually; if some succeed and some fail,
// SendMessages will return an error.
SendMessages(msgs []*ProducerMessage) error
// Close shuts down the producer; you must call this function before a producer
// object passes out of scope, as it may otherwise leak memory.
// You must call this before calling Close on the underlying client.
Close() error
// TxnStatus return current producer transaction status.
TxnStatus() ProducerTxnStatusFlag
// IsTransactional return true when current producer is transactional.
IsTransactional() bool
// BeginTxn mark current transaction as ready.
BeginTxn() error
// CommitTxn commit current transaction.
CommitTxn() error
// AbortTxn abort current transaction.
AbortTxn() error
// AddOffsetsToTxn add associated offsets to current transaction.
AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error
// AddMessageToTxn add message offsets to current transaction.
AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error
}
type syncProducer struct {
producer *asyncProducer
wg sync.WaitGroup
}
// NewSyncProducer creates a new SyncProducer using the given broker addresses and configuration.
func NewSyncProducer(addrs []string, config *Config) (SyncProducer, error) {
if config == nil {
config = NewConfig()
config.Producer.Return.Successes = true
}
if err := verifyProducerConfig(config); err != nil {
return nil, err
}
p, err := NewAsyncProducer(addrs, config)
if err != nil {
return nil, err
}
return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil
}
// NewSyncProducerFromClient creates a new SyncProducer using the given client. It is still
// necessary to call Close() on the underlying client when shutting down this producer.
func NewSyncProducerFromClient(client Client) (SyncProducer, error) {
if err := verifyProducerConfig(client.Config()); err != nil {
return nil, err
}
p, err := NewAsyncProducerFromClient(client)
if err != nil {
return nil, err
}
return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil
}
func newSyncProducerFromAsyncProducer(p *asyncProducer) *syncProducer {
sp := &syncProducer{producer: p}
sp.wg.Add(2)
go withRecover(sp.handleSuccesses)
go withRecover(sp.handleErrors)
return sp
}
func verifyProducerConfig(config *Config) error {
if !config.Producer.Return.Errors {
return ConfigurationError("Producer.Return.Errors must be true to be used in a SyncProducer")
}
if !config.Producer.Return.Successes {
return ConfigurationError("Producer.Return.Successes must be true to be used in a SyncProducer")
}
return nil
}
func (sp *syncProducer) SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) {
expectation := make(chan *ProducerError, 1)
msg.expectation = expectation
sp.producer.Input() <- msg
if pErr := <-expectation; pErr != nil {
return -1, -1, pErr.Err
}
return msg.Partition, msg.Offset, nil
}
func (sp *syncProducer) SendMessages(msgs []*ProducerMessage) error {
expectations := make(chan chan *ProducerError, len(msgs))
go func() {
for _, msg := range msgs {
expectation := make(chan *ProducerError, 1)
msg.expectation = expectation
sp.producer.Input() <- msg
expectations <- expectation
}
close(expectations)
}()
var errors ProducerErrors
for expectation := range expectations {
if pErr := <-expectation; pErr != nil {
errors = append(errors, pErr)
}
}
if len(errors) > 0 {
return errors
}
return nil
}
func (sp *syncProducer) handleSuccesses() {
defer sp.wg.Done()
for msg := range sp.producer.Successes() {
expectation := msg.expectation
expectation <- nil
}
}
func (sp *syncProducer) handleErrors() {
defer sp.wg.Done()
for err := range sp.producer.Errors() {
expectation := err.Msg.expectation
expectation <- err
}
}
func (sp *syncProducer) Close() error {
sp.producer.AsyncClose()
sp.wg.Wait()
return nil
}
func (sp *syncProducer) IsTransactional() bool {
return sp.producer.IsTransactional()
}
func (sp *syncProducer) BeginTxn() error {
return sp.producer.BeginTxn()
}
func (sp *syncProducer) CommitTxn() error {
return sp.producer.CommitTxn()
}
func (sp *syncProducer) AbortTxn() error {
return sp.producer.AbortTxn()
}
func (sp *syncProducer) AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error {
return sp.producer.AddOffsetsToTxn(offsets, groupId)
}
func (sp *syncProducer) AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error {
return sp.producer.AddMessageToTxn(msg, groupId, metadata)
}
func (p *syncProducer) TxnStatus() ProducerTxnStatusFlag {
return p.producer.TxnStatus()
}
golang-github-ibm-sarama-1.43.2/sync_producer_test.go 0000664 0000000 0000000 00000022144 14612567413 0022620 0 ustar 00root root 0000000 0000000 package sarama
import (
"errors"
"log"
"sync"
"testing"
)
func TestSyncProducer(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
leader := NewMockBroker(t, 2)
metadataResponse := new(MetadataResponse)
metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)
seedBroker.Returns(metadataResponse)
prodSuccess := new(ProduceResponse)
prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
for i := 0; i < 10; i++ {
leader.Returns(prodSuccess)
}
config := NewTestConfig()
config.Producer.Return.Successes = true
producer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 10; i++ {
msg := &ProducerMessage{
Topic: "my_topic",
Value: StringEncoder(TestMessage),
Metadata: "test",
}
partition, offset, err := producer.SendMessage(msg)
if partition != 0 || msg.Partition != partition {
t.Error("Unexpected partition")
}
if offset != 0 || msg.Offset != offset {
t.Error("Unexpected offset")
}
if str, ok := msg.Metadata.(string); !ok || str != "test" {
t.Error("Unexpected metadata")
}
if err != nil {
t.Error(err)
}
}
safeClose(t, producer)
leader.Close()
seedBroker.Close()
}
func TestSyncProducerTransactional(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
defer seedBroker.Close()
leader := NewMockBroker(t, 2)
defer leader.Close()
config := NewTestConfig()
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Producer.Return.Successes = true
config.Producer.Transaction.ID = "test"
config.Producer.Idempotent = true
config.Producer.Retry.Max = 5
config.Net.MaxOpenRequests = 1
metadataResponse := new(MetadataResponse)
metadataResponse.Version = 4
metadataResponse.ControllerID = leader.BrokerID()
metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
metadataResponse.AddTopic("my_topic", ErrNoError)
metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)
seedBroker.Returns(metadataResponse)
client, err := NewClient([]string{seedBroker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
defer safeClose(t, client)
findCoordinatorResponse := new(FindCoordinatorResponse)
findCoordinatorResponse.Coordinator = client.Brokers()[0]
findCoordinatorResponse.Version = 1
leader.Returns(findCoordinatorResponse)
initProducerIdResponse := new(InitProducerIDResponse)
leader.Returns(initProducerIdResponse)
addPartitionToTxn := new(AddPartitionsToTxnResponse)
addPartitionToTxn.Errors = map[string][]*PartitionError{
"my_topic": {
{
Partition: 0,
},
},
}
leader.Returns(addPartitionToTxn)
prodSuccess := new(ProduceResponse)
prodSuccess.Version = 3
prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
for i := 0; i < 10; i++ {
leader.Returns(prodSuccess)
}
endTxnResponse := &EndTxnResponse{}
leader.Returns(endTxnResponse)
producer, err := NewSyncProducerFromClient(client)
if err != nil {
t.Fatal(err)
}
if !producer.IsTransactional() {
t.Error("producer is not transactional")
}
err = producer.BeginTxn()
if err != nil {
t.Fatal(err)
}
if producer.TxnStatus()&ProducerTxnFlagInTransaction == 0 {
t.Error("transaction must started")
}
for i := 0; i < 10; i++ {
msg := &ProducerMessage{
Topic: "my_topic",
Value: StringEncoder(TestMessage),
Metadata: "test",
}
partition, offset, err := producer.SendMessage(msg)
if partition != 0 || msg.Partition != partition {
t.Error("Unexpected partition")
}
if offset != 0 || msg.Offset != offset {
t.Error("Unexpected offset")
}
if str, ok := msg.Metadata.(string); !ok || str != "test" {
t.Error("Unexpected metadata")
}
if err != nil {
t.Error(err)
}
}
err = producer.CommitTxn()
if err != nil {
t.Fatal(err)
}
safeClose(t, producer)
}
func TestSyncProducerBatch(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
leader := NewMockBroker(t, 2)
metadataResponse := new(MetadataResponse)
metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)
seedBroker.Returns(metadataResponse)
prodSuccess := new(ProduceResponse)
prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
leader.Returns(prodSuccess)
config := NewTestConfig()
config.Producer.Flush.Messages = 3
config.Producer.Return.Successes = true
producer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
err = producer.SendMessages([]*ProducerMessage{
{
Topic: "my_topic",
Value: StringEncoder(TestMessage),
Metadata: "test",
},
{
Topic: "my_topic",
Value: StringEncoder(TestMessage),
Metadata: "test",
},
{
Topic: "my_topic",
Value: StringEncoder(TestMessage),
Metadata: "test",
},
})
if err != nil {
t.Error(err)
}
safeClose(t, producer)
leader.Close()
seedBroker.Close()
}
func TestConcurrentSyncProducer(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
leader := NewMockBroker(t, 2)
metadataResponse := new(MetadataResponse)
metadataResponse.AddBroker(leader.Addr(), leader.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)
seedBroker.Returns(metadataResponse)
prodSuccess := new(ProduceResponse)
prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
leader.Returns(prodSuccess)
config := NewTestConfig()
config.Producer.Flush.Messages = 100
config.Producer.Return.Successes = true
producer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
wg := sync.WaitGroup{}
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
msg := &ProducerMessage{Topic: "my_topic", Value: StringEncoder(TestMessage)}
partition, _, err := producer.SendMessage(msg)
if partition != 0 {
t.Error("Unexpected partition")
}
if err != nil {
t.Error(err)
}
wg.Done()
}()
}
wg.Wait()
safeClose(t, producer)
leader.Close()
seedBroker.Close()
}
func TestSyncProducerToNonExistingTopic(t *testing.T) {
broker := NewMockBroker(t, 1)
metadataResponse := new(MetadataResponse)
metadataResponse.AddBroker(broker.Addr(), broker.BrokerID())
metadataResponse.AddTopicPartition("my_topic", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)
broker.Returns(metadataResponse)
config := NewTestConfig()
config.Metadata.Retry.Max = 0
config.Producer.Retry.Max = 0
config.Producer.Return.Successes = true
producer, err := NewSyncProducer([]string{broker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
metadataResponse = new(MetadataResponse)
metadataResponse.AddBroker(broker.Addr(), broker.BrokerID())
metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition)
broker.Returns(metadataResponse)
_, _, err = producer.SendMessage(&ProducerMessage{Topic: "unknown"})
if !errors.Is(err, ErrUnknownTopicOrPartition) {
t.Error("Uxpected ErrUnknownTopicOrPartition, found:", err)
}
safeClose(t, producer)
broker.Close()
}
func TestSyncProducerRecoveryWithRetriesDisabled(t *testing.T) {
seedBroker := NewMockBroker(t, 1)
leader1 := NewMockBroker(t, 2)
leader2 := NewMockBroker(t, 3)
metadataLeader1 := new(MetadataResponse)
metadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID())
metadataLeader1.AddTopicPartition("my_topic", 0, leader1.BrokerID(), nil, nil, nil, ErrNoError)
seedBroker.Returns(metadataLeader1)
config := NewTestConfig()
config.Producer.Retry.Max = 0 // disable!
config.Producer.Retry.Backoff = 0
config.Producer.Return.Successes = true
producer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)
if err != nil {
t.Fatal(err)
}
seedBroker.Close()
prodNotLeader := new(ProduceResponse)
prodNotLeader.AddTopicPartition("my_topic", 0, ErrNotLeaderForPartition)
leader1.Returns(prodNotLeader)
_, _, err = producer.SendMessage(&ProducerMessage{Topic: "my_topic", Value: StringEncoder(TestMessage)})
if !errors.Is(err, ErrNotLeaderForPartition) {
t.Fatal(err)
}
metadataLeader2 := new(MetadataResponse)
metadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID())
metadataLeader2.AddTopicPartition("my_topic", 0, leader2.BrokerID(), nil, nil, nil, ErrNoError)
leader1.Returns(metadataLeader2)
prodSuccess := new(ProduceResponse)
prodSuccess.AddTopicPartition("my_topic", 0, ErrNoError)
leader2.Returns(prodSuccess)
_, _, err = producer.SendMessage(&ProducerMessage{Topic: "my_topic", Value: StringEncoder(TestMessage)})
if err != nil {
t.Fatal(err)
}
leader1.Close()
leader2.Close()
safeClose(t, producer)
}
// This example shows the basic usage pattern of the SyncProducer.
func ExampleSyncProducer() {
producer, err := NewSyncProducer([]string{"localhost:9092"}, nil)
if err != nil {
log.Fatalln(err)
}
defer func() {
if err := producer.Close(); err != nil {
log.Fatalln(err)
}
}()
msg := &ProducerMessage{Topic: "my_topic", Value: StringEncoder("testing 123")}
partition, offset, err := producer.SendMessage(msg)
if err != nil {
log.Printf("FAILED to send message: %s\n", err)
} else {
log.Printf("> message sent to partition %d at offset %d\n", partition, offset)
}
}
golang-github-ibm-sarama-1.43.2/timestamp.go 0000664 0000000 0000000 00000001350 14612567413 0020701 0 ustar 00root root 0000000 0000000 package sarama
import (
"fmt"
"time"
)
type Timestamp struct {
*time.Time
}
func (t Timestamp) encode(pe packetEncoder) error {
timestamp := int64(-1)
if !t.Before(time.Unix(0, 0)) {
timestamp = t.UnixNano() / int64(time.Millisecond)
} else if !t.IsZero() {
return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", t)}
}
pe.putInt64(timestamp)
return nil
}
func (t Timestamp) decode(pd packetDecoder) error {
millis, err := pd.getInt64()
if err != nil {
return err
}
// negative timestamps are invalid, in these cases we should return
// a zero time
timestamp := time.Time{}
if millis >= 0 {
timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond))
}
*t.Time = timestamp
return nil
}
golang-github-ibm-sarama-1.43.2/tools/ 0000775 0000000 0000000 00000000000 14612567413 0017510 5 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/README.md 0000664 0000000 0000000 00000001613 14612567413 0020770 0 ustar 00root root 0000000 0000000 # Sarama tools
This folder contains applications that are useful for exploration of your Kafka cluster, or instrumentation.
Some of these tools mirror tools that ship with Kafka, but these tools won't require installing the JVM to function.
- [kafka-console-producer](./kafka-console-producer): a command line tool to produce a single message to your Kafka custer.
- [kafka-console-partitionconsumer](./kafka-console-partitionconsumer): (deprecated) a command line tool to consume a single partition of a topic on your Kafka cluster.
- [kafka-console-consumer](./kafka-console-consumer): a command line tool to consume arbitrary partitions of a topic on your Kafka cluster.
- [kafka-producer-performance](./kafka-producer-performance): a command line tool to performance test producers (sync and async) on your Kafka cluster.
To install all tools, run `go install github.com/IBM/sarama/tools/...@latest`
golang-github-ibm-sarama-1.43.2/tools/kafka-console-consumer/ 0000775 0000000 0000000 00000000000 14612567413 0024056 5 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/kafka-console-consumer/.gitignore 0000664 0000000 0000000 00000000063 14612567413 0026045 0 ustar 00root root 0000000 0000000 kafka-console-consumer
kafka-console-consumer.test
golang-github-ibm-sarama-1.43.2/tools/kafka-console-consumer/README.md 0000664 0000000 0000000 00000001660 14612567413 0025340 0 ustar 00root root 0000000 0000000 # kafka-console-consumer
A simple command line tool to consume partitions of a topic and print the
messages on the standard output.
### Installation
go get github.com/IBM/sarama/tools/kafka-console-consumer
### Usage
# Minimum invocation
kafka-console-consumer -topic=test -brokers=kafka1:9092
# It will pick up a KAFKA_PEERS environment variable
export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092
kafka-console-consumer -topic=test
# You can specify the offset you want to start at. It can be either
# `oldest`, `newest`. The default is `newest`.
kafka-console-consumer -topic=test -offset=oldest
kafka-console-consumer -topic=test -offset=newest
# You can specify the partition(s) you want to consume as a comma-separated
# list. The default is `all`.
kafka-console-consumer -topic=test -partitions=1,2,3
# Display all command line options
kafka-console-consumer -help
golang-github-ibm-sarama-1.43.2/tools/kafka-console-consumer/kafka-console-consumer.go 0000664 0000000 0000000 00000010510 14612567413 0030750 0 ustar 00root root 0000000 0000000 package main
import (
"flag"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"sync"
"syscall"
"github.com/IBM/sarama"
"github.com/IBM/sarama/tools/tls"
)
var (
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
topic = flag.String("topic", "", "REQUIRED: the topic to consume")
partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers")
offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`")
verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging")
tlsEnabled = flag.Bool("tls-enabled", false, "Whether to enable TLS")
tlsSkipVerify = flag.Bool("tls-skip-verify", false, "Whether skip TLS server cert verification")
tlsClientCert = flag.String("tls-client-cert", "", "Client cert for client authentication (use with -tls-enabled and -tls-client-key)")
tlsClientKey = flag.String("tls-client-key", "", "Client key for client authentication (use with tls-enabled and -tls-client-cert)")
bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.")
logger = log.New(os.Stderr, "", log.LstdFlags)
)
func main() {
flag.Parse()
if *brokerList == "" {
printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
}
if *topic == "" {
printUsageErrorAndExit("-topic is required")
}
if *verbose {
sarama.Logger = logger
}
var initialOffset int64
switch *offset {
case "oldest":
initialOffset = sarama.OffsetOldest
case "newest":
initialOffset = sarama.OffsetNewest
default:
printUsageErrorAndExit("-offset should be `oldest` or `newest`")
}
config := sarama.NewConfig()
if *tlsEnabled {
tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)
if err != nil {
printErrorAndExit(69, "Failed to create TLS config: %s", err)
}
config.Net.TLS.Enable = true
config.Net.TLS.Config = tlsConfig
config.Net.TLS.Config.InsecureSkipVerify = *tlsSkipVerify
}
c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), config)
if err != nil {
printErrorAndExit(69, "Failed to start consumer: %s", err)
}
partitionList, err := getPartitions(c)
if err != nil {
printErrorAndExit(69, "Failed to get the list of partitions: %s", err)
}
var (
messages = make(chan *sarama.ConsumerMessage, *bufferSize)
closing = make(chan struct{})
wg sync.WaitGroup
)
go func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGTERM, os.Interrupt)
<-signals
logger.Println("Initiating shutdown of consumer...")
close(closing)
}()
for _, partition := range partitionList {
pc, err := c.ConsumePartition(*topic, partition, initialOffset)
if err != nil {
printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err)
}
go func(pc sarama.PartitionConsumer) {
<-closing
pc.AsyncClose()
}(pc)
wg.Add(1)
go func(pc sarama.PartitionConsumer) {
defer wg.Done()
for message := range pc.Messages() {
messages <- message
}
}(pc)
}
go func() {
for msg := range messages {
fmt.Printf("Partition:\t%d\n", msg.Partition)
fmt.Printf("Offset:\t%d\n", msg.Offset)
fmt.Printf("Key:\t%s\n", string(msg.Key))
fmt.Printf("Value:\t%s\n", string(msg.Value))
fmt.Println()
}
}()
wg.Wait()
logger.Println("Done consuming topic", *topic)
close(messages)
if err := c.Close(); err != nil {
logger.Println("Failed to close consumer: ", err)
}
}
func getPartitions(c sarama.Consumer) ([]int32, error) {
if *partitions == "all" {
return c.Partitions(*topic)
}
tmp := strings.Split(*partitions, ",")
var pList []int32
for i := range tmp {
val, err := strconv.ParseInt(tmp[i], 10, 32)
if err != nil {
return nil, err
}
pList = append(pList, int32(val))
}
return pList, nil
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
func printUsageErrorAndExit(format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}
golang-github-ibm-sarama-1.43.2/tools/kafka-console-partitionconsumer/ 0000775 0000000 0000000 00000000000 14612567413 0026010 5 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/kafka-console-partitionconsumer/.gitignore 0000664 0000000 0000000 00000000105 14612567413 0027774 0 ustar 00root root 0000000 0000000 kafka-console-partitionconsumer
kafka-console-partitionconsumer.test
golang-github-ibm-sarama-1.43.2/tools/kafka-console-partitionconsumer/README.md 0000664 0000000 0000000 00000001742 14612567413 0027273 0 ustar 00root root 0000000 0000000 # kafka-console-partitionconsumer
NOTE: this tool is deprecated in favour of the more general and more powerful
`kafka-console-consumer`.
A simple command line tool to consume a partition of a topic and print the messages
on the standard output.
### Installation
go get github.com/IBM/sarama/tools/kafka-console-partitionconsumer
### Usage
# Minimum invocation
kafka-console-partitionconsumer -topic=test -partition=4 -brokers=kafka1:9092
# It will pick up a KAFKA_PEERS environment variable
export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092
kafka-console-partitionconsumer -topic=test -partition=4
# You can specify the offset you want to start at. It can be either
# `oldest`, `newest`, or a specific offset number
kafka-console-partitionconsumer -topic=test -partition=3 -offset=oldest
kafka-console-partitionconsumer -topic=test -partition=2 -offset=1337
# Display all command line options
kafka-console-partitionconsumer -help
kafka-console-partitionconsumer.go 0000664 0000000 0000000 00000005017 14612567413 0034563 0 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/kafka-console-partitionconsumer package main
import (
"flag"
"fmt"
"log"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"github.com/IBM/sarama"
)
var (
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
topic = flag.String("topic", "", "REQUIRED: the topic to consume")
partition = flag.Int("partition", -1, "REQUIRED: the partition to consume")
offset = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`, or an actual offset")
verbose = flag.Bool("verbose", false, "Whether to turn on sarama logging")
logger = log.New(os.Stderr, "", log.LstdFlags)
)
func main() {
flag.Parse()
if *brokerList == "" {
printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
}
if *topic == "" {
printUsageErrorAndExit("-topic is required")
}
if *partition == -1 {
printUsageErrorAndExit("-partition is required")
}
if *verbose {
sarama.Logger = logger
}
var (
initialOffset int64
offsetError error
)
switch *offset {
case "oldest":
initialOffset = sarama.OffsetOldest
case "newest":
initialOffset = sarama.OffsetNewest
default:
initialOffset, offsetError = strconv.ParseInt(*offset, 10, 64)
}
if offsetError != nil {
printUsageErrorAndExit("Invalid initial offset: %s", *offset)
}
c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil)
if err != nil {
printErrorAndExit(69, "Failed to start consumer: %s", err)
}
pc, err := c.ConsumePartition(*topic, int32(*partition), initialOffset)
if err != nil {
printErrorAndExit(69, "Failed to start partition consumer: %s", err)
}
go func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGTERM, os.Interrupt)
<-signals
pc.AsyncClose()
}()
for msg := range pc.Messages() {
fmt.Printf("Offset:\t%d\n", msg.Offset)
fmt.Printf("Key:\t%s\n", string(msg.Key))
fmt.Printf("Value:\t%s\n", string(msg.Value))
fmt.Println()
}
if err := c.Close(); err != nil {
logger.Println("Failed to close consumer: ", err)
}
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
func printUsageErrorAndExit(format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}
golang-github-ibm-sarama-1.43.2/tools/kafka-console-producer/ 0000775 0000000 0000000 00000000000 14612567413 0024046 5 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/kafka-console-producer/.gitignore 0000664 0000000 0000000 00000000063 14612567413 0026035 0 ustar 00root root 0000000 0000000 kafka-console-producer
kafka-console-producer.test
golang-github-ibm-sarama-1.43.2/tools/kafka-console-producer/README.md 0000664 0000000 0000000 00000002141 14612567413 0025323 0 ustar 00root root 0000000 0000000 # kafka-console-producer
A simple command line tool to produce a single message to Kafka.
### Installation
go get github.com/IBM/sarama/tools/kafka-console-producer
### Usage
# Minimum invocation
kafka-console-producer -topic=test -value=value -brokers=kafka1:9092
# It will pick up a KAFKA_PEERS environment variable
export KAFKA_PEERS=kafka1:9092,kafka2:9092,kafka3:9092
kafka-console-producer -topic=test -value=value
# It will read the value from stdin by using pipes
echo "hello world" | kafka-console-producer -topic=test
# Specify a key:
echo "hello world" | kafka-console-producer -topic=test -key=key
# Partitioning: by default, kafka-console-producer will partition as follows:
# - manual partitioning if a -partition is provided
# - hash partitioning by key if a -key is provided
# - random partitioning otherwise.
#
# You can override this using the -partitioner argument:
echo "hello world" | kafka-console-producer -topic=test -key=key -partitioner=random
# Display all command line options
kafka-console-producer -help
golang-github-ibm-sarama-1.43.2/tools/kafka-console-producer/kafka-console-producer.go 0000664 0000000 0000000 00000012136 14612567413 0030736 0 ustar 00root root 0000000 0000000 package main
import (
"flag"
"fmt"
"io"
"log"
"os"
"strings"
"github.com/rcrowley/go-metrics"
"github.com/IBM/sarama"
"github.com/IBM/sarama/tools/tls"
)
var (
brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster. You can also set the KAFKA_PEERS environment variable")
headers = flag.String("headers", "", "The headers of the message to produce. Example: -headers=foo:bar,bar:foo")
topic = flag.String("topic", "", "REQUIRED: the topic to produce to")
key = flag.String("key", "", "The key of the message to produce. Can be empty.")
value = flag.String("value", "", "REQUIRED: the value of the message to produce. You can also provide the value on stdin.")
partitioner = flag.String("partitioner", "", "The partitioning scheme to use. Can be `hash`, `manual`, or `random`")
partition = flag.Int("partition", -1, "The partition to produce to.")
verbose = flag.Bool("verbose", false, "Turn on sarama logging to stderr")
showMetrics = flag.Bool("metrics", false, "Output metrics on successful publish to stderr")
silent = flag.Bool("silent", false, "Turn off printing the message's topic, partition, and offset to stdout")
tlsEnabled = flag.Bool("tls-enabled", false, "Whether to enable TLS")
tlsSkipVerify = flag.Bool("tls-skip-verify", false, "Whether skip TLS server cert verification")
tlsClientCert = flag.String("tls-client-cert", "", "Client cert for client authentication (use with -tls-enabled and -tls-client-key)")
tlsClientKey = flag.String("tls-client-key", "", "Client key for client authentication (use with tls-enabled and -tls-client-cert)")
logger = log.New(os.Stderr, "", log.LstdFlags)
)
func main() {
flag.Parse()
if *brokerList == "" {
printUsageErrorAndExit("no -brokers specified. Alternatively, set the KAFKA_PEERS environment variable")
}
if *topic == "" {
printUsageErrorAndExit("no -topic specified")
}
if *verbose {
sarama.Logger = logger
}
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForAll
config.Producer.Return.Successes = true
if *tlsEnabled {
tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)
if err != nil {
printErrorAndExit(69, "Failed to create TLS config: %s", err)
}
config.Net.TLS.Enable = true
config.Net.TLS.Config = tlsConfig
config.Net.TLS.Config.InsecureSkipVerify = *tlsSkipVerify
}
switch *partitioner {
case "":
if *partition >= 0 {
config.Producer.Partitioner = sarama.NewManualPartitioner
} else {
config.Producer.Partitioner = sarama.NewHashPartitioner
}
case "hash":
config.Producer.Partitioner = sarama.NewHashPartitioner
case "random":
config.Producer.Partitioner = sarama.NewRandomPartitioner
case "manual":
config.Producer.Partitioner = sarama.NewManualPartitioner
if *partition == -1 {
printUsageErrorAndExit("-partition is required when partitioning manually")
}
default:
printUsageErrorAndExit(fmt.Sprintf("Partitioner %s not supported.", *partitioner))
}
message := &sarama.ProducerMessage{Topic: *topic, Partition: int32(*partition)}
if *key != "" {
message.Key = sarama.StringEncoder(*key)
}
if *value != "" {
message.Value = sarama.StringEncoder(*value)
} else if stdinAvailable() {
bytes, err := io.ReadAll(os.Stdin)
if err != nil {
printErrorAndExit(66, "Failed to read data from the standard input: %s", err)
}
message.Value = sarama.ByteEncoder(bytes)
} else {
printUsageErrorAndExit("-value is required, or you have to provide the value on stdin")
}
if *headers != "" {
var hdrs []sarama.RecordHeader
arrHdrs := strings.Split(*headers, ",")
for _, h := range arrHdrs {
if header := strings.Split(h, ":"); len(header) != 2 {
printUsageErrorAndExit("-header should be key:value. Example: -headers=foo:bar,bar:foo")
} else {
hdrs = append(hdrs, sarama.RecordHeader{
Key: []byte(header[0]),
Value: []byte(header[1]),
})
}
}
if len(hdrs) != 0 {
message.Headers = hdrs
}
}
producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config)
if err != nil {
printErrorAndExit(69, "Failed to open Kafka producer: %s", err)
}
defer func() {
if err := producer.Close(); err != nil {
logger.Println("Failed to close Kafka producer cleanly:", err)
}
}()
partition, offset, err := producer.SendMessage(message)
if err != nil {
printErrorAndExit(69, "Failed to produce message: %s", err)
} else if !*silent {
fmt.Printf("topic=%s\tpartition=%d\toffset=%d\n", *topic, partition, offset)
}
if *showMetrics {
metrics.WriteOnce(config.MetricRegistry, os.Stderr)
}
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
func printUsageErrorAndExit(message string) {
fmt.Fprintln(os.Stderr, "ERROR:", message)
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}
func stdinAvailable() bool {
stat, _ := os.Stdin.Stat()
return (stat.Mode() & os.ModeCharDevice) == 0
}
golang-github-ibm-sarama-1.43.2/tools/kafka-producer-performance/ 0000775 0000000 0000000 00000000000 14612567413 0024705 5 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/kafka-producer-performance/README.md 0000664 0000000 0000000 00000000623 14612567413 0026165 0 ustar 00root root 0000000 0000000 # kafka-producer-performance
A command line tool to test producer performance.
### Installation
go get github.com/IBM/sarama/tools/kafka-producer-performance
### Usage
# Display all command line options
kafka-producer-performance -help
# Minimum invocation
kafka-producer-performance \
-brokers=kafka:9092 \
-message-load=50000 \
-message-size=100 \
-topic=producer_test
golang-github-ibm-sarama-1.43.2/tools/kafka-producer-performance/main.go 0000664 0000000 0000000 00000031365 14612567413 0026170 0 ustar 00root root 0000000 0000000 package main
import (
"context"
"crypto/rand"
"crypto/x509"
"flag"
"fmt"
"io"
"log"
"os"
"strings"
gosync "sync"
"time"
"github.com/rcrowley/go-metrics"
"github.com/IBM/sarama"
"github.com/IBM/sarama/tools/tls"
)
var (
sync = flag.Bool(
"sync",
false,
"Use a synchronous producer.",
)
messageLoad = flag.Int(
"message-load",
0,
"REQUIRED: The number of messages to produce to -topic.",
)
messageSize = flag.Int(
"message-size",
0,
"REQUIRED: The approximate size (in bytes) of each message to produce to -topic.",
)
brokers = flag.String(
"brokers",
"",
"REQUIRED: A comma separated list of broker addresses.",
)
securityProtocol = flag.String(
"security-protocol",
"PLAINTEXT",
"The name of the security protocol to talk to Kafka (PLAINTEXT, SSL).",
)
tlsRootCACerts = flag.String(
"tls-ca-certs",
"",
"The path to a file that contains a set of root certificate authorities in PEM format "+
"to trust when verifying broker certificates when -security-protocol=SSL "+
"(leave empty to use the host's root CA set).",
)
tlsClientCert = flag.String(
"tls-client-cert",
"",
"The path to a file that contains the client certificate to send to the broker "+
"in PEM format if client authentication is required when -security-protocol=SSL "+
"(leave empty to disable client authentication).",
)
tlsClientKey = flag.String(
"tls-client-key",
"",
"The path to a file that contains the client private key linked to the client certificate "+
"in PEM format when -security-protocol=SSL (REQUIRED if tls-client-cert is provided).",
)
topic = flag.String(
"topic",
"",
"REQUIRED: The topic to run the performance test on.",
)
partition = flag.Int(
"partition",
-1,
"The partition of -topic to run the performance test on.",
)
throughput = flag.Int(
"throughput",
0,
"The maximum number of messages to send per second (0 for no limit).",
)
maxOpenRequests = flag.Int(
"max-open-requests",
5,
"The maximum number of unacknowledged requests the client will send on a single connection before blocking.",
)
maxMessageBytes = flag.Int(
"max-message-bytes",
1000000,
"The max permitted size of a message.",
)
requiredAcks = flag.Int(
"required-acks",
1,
"The required number of acks needed from the broker (-1: all, 0: none, 1: local).",
)
timeout = flag.Duration(
"timeout",
10*time.Second,
"The duration the producer will wait to receive -required-acks.",
)
partitioner = flag.String(
"partitioner",
"roundrobin",
"The partitioning scheme to use (hash, manual, random, roundrobin).",
)
compression = flag.String(
"compression",
"none",
"The compression method to use (none, gzip, snappy, lz4).",
)
flushFrequency = flag.Duration(
"flush-frequency",
0,
"The best-effort frequency of flushes.",
)
flushBytes = flag.Int(
"flush-bytes",
0,
"The best-effort number of bytes needed to trigger a flush.",
)
flushMessages = flag.Int(
"flush-messages",
0,
"The best-effort number of messages needed to trigger a flush.",
)
flushMaxMessages = flag.Int(
"flush-max-messages",
0,
"The maximum number of messages the producer will send in a single request.",
)
clientID = flag.String(
"client-id",
"sarama",
"The client ID sent with every request to the brokers.",
)
channelBufferSize = flag.Int(
"channel-buffer-size",
256,
"The number of events to buffer in internal and external channels.",
)
routines = flag.Int(
"routines",
1,
"The number of routines to send the messages from (-sync only).",
)
version = flag.String(
"version",
"0.8.2.0",
"The assumed version of Kafka.",
)
verbose = flag.Bool(
"verbose",
false,
"Turn on sarama logging to stderr",
)
)
func parseCompression(scheme string) sarama.CompressionCodec {
switch scheme {
case "none":
return sarama.CompressionNone
case "gzip":
return sarama.CompressionGZIP
case "snappy":
return sarama.CompressionSnappy
case "lz4":
return sarama.CompressionLZ4
default:
printUsageErrorAndExit(fmt.Sprintf("Unknown -compression: %s", scheme))
}
panic("should not happen")
}
func parsePartitioner(scheme string, partition int) sarama.PartitionerConstructor {
if partition < 0 && scheme == "manual" {
printUsageErrorAndExit("-partition must not be -1 for -partitioning=manual")
}
switch scheme {
case "manual":
return sarama.NewManualPartitioner
case "hash":
return sarama.NewHashPartitioner
case "random":
return sarama.NewRandomPartitioner
case "roundrobin":
return sarama.NewRoundRobinPartitioner
default:
printUsageErrorAndExit(fmt.Sprintf("Unknown -partitioning: %s", scheme))
}
panic("should not happen")
}
func parseVersion(version string) sarama.KafkaVersion {
result, err := sarama.ParseKafkaVersion(version)
if err != nil {
printUsageErrorAndExit(fmt.Sprintf("unknown -version: %s", version))
}
return result
}
func generateMessages(topic string, partition, messageLoad, messageSize int) []*sarama.ProducerMessage {
messages := make([]*sarama.ProducerMessage, messageLoad)
for i := 0; i < messageLoad; i++ {
payload := make([]byte, messageSize)
if _, err := rand.Read(payload); err != nil {
printErrorAndExit(69, "Failed to generate message payload: %s", err)
}
messages[i] = &sarama.ProducerMessage{
Topic: topic,
Partition: int32(partition),
Value: sarama.ByteEncoder(payload),
}
}
return messages
}
func main() {
flag.Parse()
if *brokers == "" {
printUsageErrorAndExit("-brokers is required")
}
if *topic == "" {
printUsageErrorAndExit("-topic is required")
}
if *messageLoad <= 0 {
printUsageErrorAndExit("-message-load must be greater than 0")
}
if *messageSize <= 0 {
printUsageErrorAndExit("-message-size must be greater than 0")
}
if *routines < 1 || *routines > *messageLoad {
printUsageErrorAndExit("-routines must be greater than 0 and less than or equal to -message-load")
}
if *securityProtocol != "PLAINTEXT" && *securityProtocol != "SSL" {
printUsageErrorAndExit(fmt.Sprintf("-security-protocol %q is not supported", *securityProtocol))
}
if *verbose {
sarama.Logger = log.New(os.Stderr, "", log.LstdFlags)
}
config := sarama.NewConfig()
config.Net.MaxOpenRequests = *maxOpenRequests
config.Producer.MaxMessageBytes = *maxMessageBytes
config.Producer.RequiredAcks = sarama.RequiredAcks(*requiredAcks)
config.Producer.Timeout = *timeout
config.Producer.Partitioner = parsePartitioner(*partitioner, *partition)
config.Producer.Compression = parseCompression(*compression)
config.Producer.Flush.Frequency = *flushFrequency
config.Producer.Flush.Bytes = *flushBytes
config.Producer.Flush.Messages = *flushMessages
config.Producer.Flush.MaxMessages = *flushMaxMessages
config.Producer.Return.Successes = true
config.ClientID = *clientID
config.ChannelBufferSize = *channelBufferSize
config.Version = parseVersion(*version)
if *securityProtocol == "SSL" {
tlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)
if err != nil {
printErrorAndExit(69, "failed to load client certificate from: %s and private key from: %s: %v",
*tlsClientCert, *tlsClientKey, err)
}
if *tlsRootCACerts != "" {
rootCAsBytes, err := os.ReadFile(*tlsRootCACerts)
if err != nil {
printErrorAndExit(69, "failed to read root CA certificates: %v", err)
}
certPool := x509.NewCertPool()
if !certPool.AppendCertsFromPEM(rootCAsBytes) {
printErrorAndExit(69, "failed to load root CA certificates from file: %s", *tlsRootCACerts)
}
// Use specific root CA set vs the host's set
tlsConfig.RootCAs = certPool
}
config.Net.TLS.Enable = true
config.Net.TLS.Config = tlsConfig
}
if err := config.Validate(); err != nil {
printErrorAndExit(69, "Invalid configuration: %s", err)
}
// Print out metrics periodically.
done := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
go func(ctx context.Context) {
defer close(done)
t := time.Tick(5 * time.Second)
for {
select {
case <-t:
printMetrics(os.Stdout, config.MetricRegistry)
case <-ctx.Done():
return
}
}
}(ctx)
brokers := strings.Split(*brokers, ",")
if *sync {
runSyncProducer(*topic, *partition, *messageLoad, *messageSize, *routines,
config, brokers, *throughput)
} else {
runAsyncProducer(*topic, *partition, *messageLoad, *messageSize,
config, brokers, *throughput)
}
cancel()
<-done
}
func runAsyncProducer(topic string, partition, messageLoad, messageSize int,
config *sarama.Config, brokers []string, throughput int) {
producer, err := sarama.NewAsyncProducer(brokers, config)
if err != nil {
printErrorAndExit(69, "Failed to create producer: %s", err)
}
defer func() {
// Print final metrics.
printMetrics(os.Stdout, config.MetricRegistry)
if err := producer.Close(); err != nil {
printErrorAndExit(69, "Failed to close producer: %s", err)
}
}()
messages := generateMessages(topic, partition, messageLoad, messageSize)
messagesDone := make(chan struct{})
go func() {
for i := 0; i < messageLoad; i++ {
select {
case <-producer.Successes():
case err = <-producer.Errors():
printErrorAndExit(69, "%s", err)
}
}
messagesDone <- struct{}{}
}()
if throughput > 0 {
ticker := time.NewTicker(time.Second)
for idx, message := range messages {
producer.Input() <- message
if (idx+1)%throughput == 0 {
<-ticker.C
}
}
ticker.Stop()
} else {
for _, message := range messages {
producer.Input() <- message
}
}
<-messagesDone
close(messagesDone)
}
func runSyncProducer(topic string, partition, messageLoad, messageSize, routines int,
config *sarama.Config, brokers []string, throughput int) {
producer, err := sarama.NewSyncProducer(brokers, config)
if err != nil {
printErrorAndExit(69, "Failed to create producer: %s", err)
}
defer func() {
// Print final metrics.
printMetrics(os.Stdout, config.MetricRegistry)
if err := producer.Close(); err != nil {
printErrorAndExit(69, "Failed to close producer: %s", err)
}
}()
messages := make([][]*sarama.ProducerMessage, routines)
for i := 0; i < routines; i++ {
if i == routines-1 {
messages[i] = generateMessages(topic, partition, messageLoad/routines+messageLoad%routines, messageSize)
} else {
messages[i] = generateMessages(topic, partition, messageLoad/routines, messageSize)
}
}
var wg gosync.WaitGroup
if throughput > 0 {
for _, messages := range messages {
messages := messages
wg.Add(1)
go func() {
ticker := time.NewTicker(time.Second)
for _, message := range messages {
for i := 0; i < throughput; i++ {
_, _, err = producer.SendMessage(message)
if err != nil {
printErrorAndExit(69, "Failed to send message: %s", err)
}
}
<-ticker.C
}
ticker.Stop()
wg.Done()
}()
}
} else {
for _, messages := range messages {
messages := messages
wg.Add(1)
go func() {
for _, message := range messages {
_, _, err = producer.SendMessage(message)
if err != nil {
printErrorAndExit(69, "Failed to send message: %s", err)
}
}
wg.Done()
}()
}
}
wg.Wait()
}
func printMetrics(w io.Writer, r metrics.Registry) {
recordSendRateMetric := r.Get("record-send-rate")
requestLatencyMetric := r.Get("request-latency-in-ms")
outgoingByteRateMetric := r.Get("outgoing-byte-rate")
requestsInFlightMetric := r.Get("requests-in-flight")
if recordSendRateMetric == nil || requestLatencyMetric == nil || outgoingByteRateMetric == nil ||
requestsInFlightMetric == nil {
return
}
recordSendRate := recordSendRateMetric.(metrics.Meter).Snapshot()
requestLatency := requestLatencyMetric.(metrics.Histogram).Snapshot()
requestLatencyPercentiles := requestLatency.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
outgoingByteRate := outgoingByteRateMetric.(metrics.Meter).Snapshot()
requestsInFlight := requestsInFlightMetric.(metrics.Counter).Count()
fmt.Fprintf(w, "%d records sent, %.1f records/sec (%.2f MiB/sec ingress, %.2f MiB/sec egress), "+
"%.1f ms avg latency, %.1f ms stddev, %.1f ms 50th, %.1f ms 75th, "+
"%.1f ms 95th, %.1f ms 99th, %.1f ms 99.9th, %d total req. in flight\n",
recordSendRate.Count(),
recordSendRate.RateMean(),
recordSendRate.RateMean()*float64(*messageSize)/1024/1024,
outgoingByteRate.RateMean()/1024/1024,
requestLatency.Mean(),
requestLatency.StdDev(),
requestLatencyPercentiles[0],
requestLatencyPercentiles[1],
requestLatencyPercentiles[2],
requestLatencyPercentiles[3],
requestLatencyPercentiles[4],
requestsInFlight,
)
}
func printUsageErrorAndExit(message string) {
fmt.Fprintln(os.Stderr, "ERROR:", message)
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "Available command line options:")
flag.PrintDefaults()
os.Exit(64)
}
func printErrorAndExit(code int, format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
fmt.Fprintln(os.Stderr)
os.Exit(code)
}
golang-github-ibm-sarama-1.43.2/tools/tls/ 0000775 0000000 0000000 00000000000 14612567413 0020312 5 ustar 00root root 0000000 0000000 golang-github-ibm-sarama-1.43.2/tools/tls/config.go 0000664 0000000 0000000 00000000611 14612567413 0022104 0 ustar 00root root 0000000 0000000 package tls
import "crypto/tls"
func NewConfig(clientCert, clientKey string) (*tls.Config, error) {
tlsConfig := tls.Config{
MinVersion: tls.VersionTLS12,
}
if clientCert != "" && clientKey != "" {
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return &tlsConfig, err
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
return &tlsConfig, nil
}
golang-github-ibm-sarama-1.43.2/transaction_manager.go 0000664 0000000 0000000 00000072362 14612567413 0022730 0 ustar 00root root 0000000 0000000 package sarama
import (
"errors"
"fmt"
"strings"
"sync"
"time"
)
// ProducerTxnStatusFlag mark current transaction status.
type ProducerTxnStatusFlag int16
const (
// ProducerTxnFlagUninitialized when txnmgr is created
ProducerTxnFlagUninitialized ProducerTxnStatusFlag = 1 << iota
// ProducerTxnFlagInitializing when txnmgr is initializing
ProducerTxnFlagInitializing
// ProducerTxnFlagReady when is ready to receive transaction
ProducerTxnFlagReady
// ProducerTxnFlagInTransaction when transaction is started
ProducerTxnFlagInTransaction
// ProducerTxnFlagEndTransaction when transaction will be committed
ProducerTxnFlagEndTransaction
// ProducerTxnFlagInError when having abortable or fatal error
ProducerTxnFlagInError
// ProducerTxnFlagCommittingTransaction when committing txn
ProducerTxnFlagCommittingTransaction
// ProducerTxnFlagAbortingTransaction when committing txn
ProducerTxnFlagAbortingTransaction
// ProducerTxnFlagAbortableError when producer encounter an abortable error
// Must call AbortTxn in this case.
ProducerTxnFlagAbortableError
// ProducerTxnFlagFatalError when producer encounter an fatal error
// Must Close an recreate it.
ProducerTxnFlagFatalError
)
func (s ProducerTxnStatusFlag) String() string {
status := make([]string, 0)
if s&ProducerTxnFlagUninitialized != 0 {
status = append(status, "ProducerTxnStateUninitialized")
}
if s&ProducerTxnFlagInitializing != 0 {
status = append(status, "ProducerTxnStateInitializing")
}
if s&ProducerTxnFlagReady != 0 {
status = append(status, "ProducerTxnStateReady")
}
if s&ProducerTxnFlagInTransaction != 0 {
status = append(status, "ProducerTxnStateInTransaction")
}
if s&ProducerTxnFlagEndTransaction != 0 {
status = append(status, "ProducerTxnStateEndTransaction")
}
if s&ProducerTxnFlagInError != 0 {
status = append(status, "ProducerTxnStateInError")
}
if s&ProducerTxnFlagCommittingTransaction != 0 {
status = append(status, "ProducerTxnStateCommittingTransaction")
}
if s&ProducerTxnFlagAbortingTransaction != 0 {
status = append(status, "ProducerTxnStateAbortingTransaction")
}
if s&ProducerTxnFlagAbortableError != 0 {
status = append(status, "ProducerTxnStateAbortableError")
}
if s&ProducerTxnFlagFatalError != 0 {
status = append(status, "ProducerTxnStateFatalError")
}
return strings.Join(status, "|")
}
// transactionManager keeps the state necessary to ensure idempotent production
type transactionManager struct {
producerID int64
producerEpoch int16
sequenceNumbers map[string]int32
mutex sync.Mutex
transactionalID string
transactionTimeout time.Duration
client Client
// when kafka cluster is at least 2.5.0.
// used to recover when producer failed.
coordinatorSupportsBumpingEpoch bool
// When producer need to bump it's epoch.
epochBumpRequired bool
// Record last seen error.
lastError error
// Ensure that status is never accessed with a race-condition.
statusLock sync.RWMutex
status ProducerTxnStatusFlag
// Ensure that only one goroutine will update partitions in current transaction.
partitionInTxnLock sync.Mutex
pendingPartitionsInCurrentTxn topicPartitionSet
partitionsInCurrentTxn topicPartitionSet
// Offsets to add to transaction.
offsetsInCurrentTxn map[string]topicPartitionOffsets
}
const (
noProducerID = -1
noProducerEpoch = -1
// see publishTxnPartitions comment.
addPartitionsRetryBackoff = 20 * time.Millisecond
)
// txnmngr allowed transitions.
var producerTxnTransitions = map[ProducerTxnStatusFlag][]ProducerTxnStatusFlag{
ProducerTxnFlagUninitialized: {
ProducerTxnFlagReady,
ProducerTxnFlagInError,
},
// When we need are initializing
ProducerTxnFlagInitializing: {
ProducerTxnFlagInitializing,
ProducerTxnFlagReady,
ProducerTxnFlagInError,
},
// When we have initialized transactional producer
ProducerTxnFlagReady: {
ProducerTxnFlagInTransaction,
},
// When beginTxn has been called
ProducerTxnFlagInTransaction: {
// When calling commit or abort
ProducerTxnFlagEndTransaction,
// When got an error
ProducerTxnFlagInError,
},
ProducerTxnFlagEndTransaction: {
// When epoch bump
ProducerTxnFlagInitializing,
// When commit is good
ProducerTxnFlagReady,
// When got an error
ProducerTxnFlagInError,
},
// Need to abort transaction
ProducerTxnFlagAbortableError: {
// Call AbortTxn
ProducerTxnFlagAbortingTransaction,
// When got an error
ProducerTxnFlagInError,
},
// Need to close producer
ProducerTxnFlagFatalError: {
ProducerTxnFlagFatalError,
},
}
type topicPartition struct {
topic string
partition int32
}
// to ensure that we don't do a full scan every time a partition or an offset is added.
type (
topicPartitionSet map[topicPartition]struct{}
topicPartitionOffsets map[topicPartition]*PartitionOffsetMetadata
)
func (s topicPartitionSet) mapToRequest() map[string][]int32 {
result := make(map[string][]int32, len(s))
for tp := range s {
result[tp.topic] = append(result[tp.topic], tp.partition)
}
return result
}
func (s topicPartitionOffsets) mapToRequest() map[string][]*PartitionOffsetMetadata {
result := make(map[string][]*PartitionOffsetMetadata, len(s))
for tp, offset := range s {
result[tp.topic] = append(result[tp.topic], offset)
}
return result
}
// Return true if current transition is allowed.
func (t *transactionManager) isTransitionValid(target ProducerTxnStatusFlag) bool {
for status, allowedTransitions := range producerTxnTransitions {
if status&t.status != 0 {
for _, allowedTransition := range allowedTransitions {
if allowedTransition&target != 0 {
return true
}
}
}
}
return false
}
// Get current transaction status.
func (t *transactionManager) currentTxnStatus() ProducerTxnStatusFlag {
t.statusLock.RLock()
defer t.statusLock.RUnlock()
return t.status
}
// Try to transition to a valid status and return an error otherwise.
func (t *transactionManager) transitionTo(target ProducerTxnStatusFlag, err error) error {
t.statusLock.Lock()
defer t.statusLock.Unlock()
if !t.isTransitionValid(target) {
return ErrTransitionNotAllowed
}
if target&ProducerTxnFlagInError != 0 {
if err == nil {
return ErrCannotTransitionNilError
}
t.lastError = err
} else {
t.lastError = nil
}
DebugLogger.Printf("txnmgr/transition [%s] transition from %s to %s\n", t.transactionalID, t.status, target)
t.status = target
return err
}
func (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) (int32, int16) {
key := fmt.Sprintf("%s-%d", topic, partition)
t.mutex.Lock()
defer t.mutex.Unlock()
sequence := t.sequenceNumbers[key]
t.sequenceNumbers[key] = sequence + 1
return sequence, t.producerEpoch
}
func (t *transactionManager) bumpEpoch() {
t.mutex.Lock()
defer t.mutex.Unlock()
t.producerEpoch++
for k := range t.sequenceNumbers {
t.sequenceNumbers[k] = 0
}
}
func (t *transactionManager) getProducerID() (int64, int16) {
t.mutex.Lock()
defer t.mutex.Unlock()
return t.producerID, t.producerEpoch
}
// Compute retry backoff considered current attempts.
func (t *transactionManager) computeBackoff(attemptsRemaining int) time.Duration {
if t.client.Config().Producer.Transaction.Retry.BackoffFunc != nil {
maxRetries := t.client.Config().Producer.Transaction.Retry.Max
retries := maxRetries - attemptsRemaining
return t.client.Config().Producer.Transaction.Retry.BackoffFunc(retries, maxRetries)
}
return t.client.Config().Producer.Transaction.Retry.Backoff
}
// return true is txnmngr is transactinal.
func (t *transactionManager) isTransactional() bool {
return t.transactionalID != ""
}
// add specified offsets to current transaction.
func (t *transactionManager) addOffsetsToTxn(offsetsToAdd map[string][]*PartitionOffsetMetadata, groupId string) error {
t.mutex.Lock()
defer t.mutex.Unlock()
if t.currentTxnStatus()&ProducerTxnFlagInTransaction == 0 {
return ErrTransactionNotReady
}
if t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {
return t.lastError
}
if _, ok := t.offsetsInCurrentTxn[groupId]; !ok {
t.offsetsInCurrentTxn[groupId] = topicPartitionOffsets{}
}
for topic, offsets := range offsetsToAdd {
for _, offset := range offsets {
tp := topicPartition{topic: topic, partition: offset.Partition}
t.offsetsInCurrentTxn[groupId][tp] = offset
}
}
return nil
}
// send txnmgnr save offsets to transaction coordinator.
func (t *transactionManager) publishOffsetsToTxn(offsets topicPartitionOffsets, groupId string) (topicPartitionOffsets, error) {
// First AddOffsetsToTxn
attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
exec := func(run func() (bool, error), err error) error {
for attemptsRemaining >= 0 {
var retry bool
retry, err = run()
if !retry {
return err
}
backoff := t.computeBackoff(attemptsRemaining)
Logger.Printf("txnmgr/add-offset-to-txn [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
time.Sleep(backoff)
attemptsRemaining--
}
return err
}
lastError := exec(func() (bool, error) {
coordinator, err := t.client.TransactionCoordinator(t.transactionalID)
if err != nil {
return true, err
}
request := &AddOffsetsToTxnRequest{
TransactionalID: t.transactionalID,
ProducerEpoch: t.producerEpoch,
ProducerID: t.producerID,
GroupID: groupId,
}
if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
// Version 2 adds the support for new error code PRODUCER_FENCED.
request.Version = 2
} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
// Version 1 is the same as version 0.
request.Version = 1
}
response, err := coordinator.AddOffsetsToTxn(request)
if err != nil {
// If an error occurred try to refresh current transaction coordinator.
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
return true, err
}
if response == nil {
// If no response is returned just retry.
return true, ErrTxnUnableToParseResponse
}
if response.Err == ErrNoError {
DebugLogger.Printf("txnmgr/add-offset-to-txn [%s] successful add-offset-to-txn with group %s %+v\n",
t.transactionalID, groupId, response)
// If no error, just exit.
return false, nil
}
switch response.Err {
case ErrConsumerCoordinatorNotAvailable:
fallthrough
case ErrNotCoordinatorForConsumer:
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
fallthrough
case ErrOffsetsLoadInProgress:
fallthrough
case ErrConcurrentTransactions:
// Retry
case ErrUnknownProducerID:
fallthrough
case ErrInvalidProducerIDMapping:
return false, t.abortableErrorIfPossible(response.Err)
case ErrGroupAuthorizationFailed:
return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, response.Err)
default:
// Others are fatal
return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
}
return true, response.Err
}, nil)
if lastError != nil {
return offsets, lastError
}
resultOffsets := offsets
// Then TxnOffsetCommit
// note the result is not completed until the TxnOffsetCommit returns
attemptsRemaining = t.client.Config().Producer.Transaction.Retry.Max
execTxnOffsetCommit := func(run func() (topicPartitionOffsets, bool, error), err error) (topicPartitionOffsets, error) {
var r topicPartitionOffsets
for attemptsRemaining >= 0 {
var retry bool
r, retry, err = run()
if !retry {
return r, err
}
backoff := t.computeBackoff(attemptsRemaining)
Logger.Printf("txnmgr/txn-offset-commit [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
time.Sleep(backoff)
attemptsRemaining--
}
return r, err
}
return execTxnOffsetCommit(func() (topicPartitionOffsets, bool, error) {
consumerGroupCoordinator, err := t.client.Coordinator(groupId)
if err != nil {
return resultOffsets, true, err
}
request := &TxnOffsetCommitRequest{
TransactionalID: t.transactionalID,
ProducerEpoch: t.producerEpoch,
ProducerID: t.producerID,
GroupID: groupId,
Topics: offsets.mapToRequest(),
}
if t.client.Config().Version.IsAtLeast(V2_1_0_0) {
// Version 2 adds the committed leader epoch.
request.Version = 2
} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
// Version 1 is the same as version 0.
request.Version = 1
}
responses, err := consumerGroupCoordinator.TxnOffsetCommit(request)
if err != nil {
_ = consumerGroupCoordinator.Close()
_ = t.client.RefreshCoordinator(groupId)
return resultOffsets, true, err
}
if responses == nil {
return resultOffsets, true, ErrTxnUnableToParseResponse
}
var responseErrors []error
failedTxn := topicPartitionOffsets{}
for topic, partitionErrors := range responses.Topics {
for _, partitionError := range partitionErrors {
switch partitionError.Err {
case ErrNoError:
continue
// If the topic is unknown or the coordinator is loading, retry with the current coordinator
case ErrRequestTimedOut:
fallthrough
case ErrConsumerCoordinatorNotAvailable:
fallthrough
case ErrNotCoordinatorForConsumer:
_ = consumerGroupCoordinator.Close()
_ = t.client.RefreshCoordinator(groupId)
fallthrough
case ErrUnknownTopicOrPartition:
fallthrough
case ErrOffsetsLoadInProgress:
// Do nothing just retry
case ErrIllegalGeneration:
fallthrough
case ErrUnknownMemberId:
fallthrough
case ErrFencedInstancedId:
fallthrough
case ErrGroupAuthorizationFailed:
return resultOffsets, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, partitionError.Err)
default:
// Others are fatal
return resultOffsets, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, partitionError.Err)
}
tp := topicPartition{topic: topic, partition: partitionError.Partition}
failedTxn[tp] = offsets[tp]
responseErrors = append(responseErrors, partitionError.Err)
}
}
resultOffsets = failedTxn
if len(resultOffsets) == 0 {
DebugLogger.Printf("txnmgr/txn-offset-commit [%s] successful txn-offset-commit with group %s %+v\n",
t.transactionalID, groupId)
return resultOffsets, false, nil
}
return resultOffsets, true, Wrap(ErrTxnOffsetCommit, responseErrors...)
}, nil)
}
func (t *transactionManager) initProducerId() (int64, int16, error) {
isEpochBump := false
req := &InitProducerIDRequest{}
if t.isTransactional() {
req.TransactionalID = &t.transactionalID
req.TransactionTimeout = t.transactionTimeout
}
if t.client.Config().Version.IsAtLeast(V2_5_0_0) {
if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
// Version 4 adds the support for new error code PRODUCER_FENCED.
req.Version = 4
} else {
// Version 3 adds ProducerId and ProducerEpoch, allowing producers to try
// to resume after an INVALID_PRODUCER_EPOCH error
req.Version = 3
}
isEpochBump = t.producerID != noProducerID && t.producerEpoch != noProducerEpoch
t.coordinatorSupportsBumpingEpoch = true
req.ProducerID = t.producerID
req.ProducerEpoch = t.producerEpoch
} else if t.client.Config().Version.IsAtLeast(V2_4_0_0) {
// Version 2 is the first flexible version.
req.Version = 2
} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
// Version 1 is the same as version 0.
req.Version = 1
}
if isEpochBump {
err := t.transitionTo(ProducerTxnFlagInitializing, nil)
if err != nil {
return -1, -1, err
}
DebugLogger.Printf("txnmgr/init-producer-id [%s] invoking InitProducerId for the first time in order to acquire a producer ID\n",
t.transactionalID)
} else {
DebugLogger.Printf("txnmgr/init-producer-id [%s] invoking InitProducerId with current producer ID %d and epoch %d in order to bump the epoch\n",
t.transactionalID, t.producerID, t.producerEpoch)
}
attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
exec := func(run func() (int64, int16, bool, error), err error) (int64, int16, error) {
pid := int64(-1)
pepoch := int16(-1)
for attemptsRemaining >= 0 {
var retry bool
pid, pepoch, retry, err = run()
if !retry {
return pid, pepoch, err
}
backoff := t.computeBackoff(attemptsRemaining)
Logger.Printf("txnmgr/init-producer-id [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
time.Sleep(backoff)
attemptsRemaining--
}
return -1, -1, err
}
return exec(func() (int64, int16, bool, error) {
var err error
var coordinator *Broker
if t.isTransactional() {
coordinator, err = t.client.TransactionCoordinator(t.transactionalID)
} else {
coordinator = t.client.LeastLoadedBroker()
}
if err != nil {
return -1, -1, true, err
}
response, err := coordinator.InitProducerID(req)
if err != nil {
if t.isTransactional() {
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
}
return -1, -1, true, err
}
if response == nil {
return -1, -1, true, ErrTxnUnableToParseResponse
}
if response.Err == ErrNoError {
if isEpochBump {
t.sequenceNumbers = make(map[string]int32)
}
err := t.transitionTo(ProducerTxnFlagReady, nil)
if err != nil {
return -1, -1, true, err
}
DebugLogger.Printf("txnmgr/init-producer-id [%s] successful init producer id %+v\n",
t.transactionalID, response)
return response.ProducerID, response.ProducerEpoch, false, nil
}
switch response.Err {
// Retriable errors
case ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer, ErrOffsetsLoadInProgress:
if t.isTransactional() {
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
}
// Fatal errors
default:
return -1, -1, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
}
return -1, -1, true, response.Err
}, nil)
}
// if kafka cluster is at least 2.5.0 mark txnmngr to bump epoch else mark it as fatal.
func (t *transactionManager) abortableErrorIfPossible(err error) error {
if t.coordinatorSupportsBumpingEpoch {
t.epochBumpRequired = true
return t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, err)
}
return t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, err)
}
// End current transaction.
func (t *transactionManager) completeTransaction() error {
if t.epochBumpRequired {
err := t.transitionTo(ProducerTxnFlagInitializing, nil)
if err != nil {
return err
}
} else {
err := t.transitionTo(ProducerTxnFlagReady, nil)
if err != nil {
return err
}
}
t.lastError = nil
t.epochBumpRequired = false
t.partitionsInCurrentTxn = topicPartitionSet{}
t.pendingPartitionsInCurrentTxn = topicPartitionSet{}
t.offsetsInCurrentTxn = map[string]topicPartitionOffsets{}
return nil
}
// send EndTxn request with commit flag. (true when committing false otherwise)
func (t *transactionManager) endTxn(commit bool) error {
attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
exec := func(run func() (bool, error), err error) error {
for attemptsRemaining >= 0 {
var retry bool
retry, err = run()
if !retry {
return err
}
backoff := t.computeBackoff(attemptsRemaining)
Logger.Printf("txnmgr/endtxn [%s] retrying after %dms... (%d attempts remaining) (%s)\n",
t.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)
time.Sleep(backoff)
attemptsRemaining--
}
return err
}
return exec(func() (bool, error) {
coordinator, err := t.client.TransactionCoordinator(t.transactionalID)
if err != nil {
return true, err
}
request := &EndTxnRequest{
TransactionalID: t.transactionalID,
ProducerEpoch: t.producerEpoch,
ProducerID: t.producerID,
TransactionResult: commit,
}
if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
// Version 2 adds the support for new error code PRODUCER_FENCED.
request.Version = 2
} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
// Version 1 is the same as version 0.
request.Version = 1
}
response, err := coordinator.EndTxn(request)
if err != nil {
// Always retry on network error
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
return true, err
}
if response == nil {
return true, ErrTxnUnableToParseResponse
}
if response.Err == ErrNoError {
DebugLogger.Printf("txnmgr/endtxn [%s] successful to end txn %+v\n",
t.transactionalID, response)
return false, t.completeTransaction()
}
switch response.Err {
// Need to refresh coordinator
case ErrConsumerCoordinatorNotAvailable:
fallthrough
case ErrNotCoordinatorForConsumer:
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
fallthrough
case ErrOffsetsLoadInProgress:
fallthrough
case ErrConcurrentTransactions:
// Just retry
case ErrUnknownProducerID:
fallthrough
case ErrInvalidProducerIDMapping:
return false, t.abortableErrorIfPossible(response.Err)
// Fatal errors
default:
return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
}
return true, response.Err
}, nil)
}
// We will try to publish associated offsets for each groups
// then send endtxn request to mark transaction as finished.
func (t *transactionManager) finishTransaction(commit bool) error {
t.mutex.Lock()
defer t.mutex.Unlock()
// Ensure no error when committing or aborting
if commit && t.currentTxnStatus()&ProducerTxnFlagInError != 0 {
return t.lastError
} else if !commit && t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {
return t.lastError
}
// if no records has been sent don't do anything.
if len(t.partitionsInCurrentTxn) == 0 {
return t.completeTransaction()
}
epochBump := t.epochBumpRequired
// If we're aborting the transaction, so there should be no need to add offsets.
if commit && len(t.offsetsInCurrentTxn) > 0 {
for group, offsets := range t.offsetsInCurrentTxn {
newOffsets, err := t.publishOffsetsToTxn(offsets, group)
if err != nil {
t.offsetsInCurrentTxn[group] = newOffsets
return err
}
delete(t.offsetsInCurrentTxn, group)
}
}
if t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {
return t.lastError
}
if !errors.Is(t.lastError, ErrInvalidProducerIDMapping) {
err := t.endTxn(commit)
if err != nil {
return err
}
if !epochBump {
return nil
}
}
// reset pid and epoch if needed.
return t.initializeTransactions()
}
// called before sending any transactional record
// won't do anything if current topic-partition is already added to transaction.
func (t *transactionManager) maybeAddPartitionToCurrentTxn(topic string, partition int32) {
if t.currentTxnStatus()&ProducerTxnFlagInError != 0 {
return
}
tp := topicPartition{topic: topic, partition: partition}
t.partitionInTxnLock.Lock()
defer t.partitionInTxnLock.Unlock()
if _, ok := t.partitionsInCurrentTxn[tp]; ok {
// partition is already added
return
}
t.pendingPartitionsInCurrentTxn[tp] = struct{}{}
}
// Makes a request to kafka to add a list of partitions ot the current transaction.
func (t *transactionManager) publishTxnPartitions() error {
t.partitionInTxnLock.Lock()
defer t.partitionInTxnLock.Unlock()
if t.currentTxnStatus()&ProducerTxnFlagInError != 0 {
return t.lastError
}
if len(t.pendingPartitionsInCurrentTxn) == 0 {
return nil
}
// Remove the partitions from the pending set regardless of the result. We use the presence
// of partitions in the pending set to know when it is not safe to send batches. However, if
// the partitions failed to be added and we enter an error state, we expect the batches to be
// aborted anyway. In this case, we must be able to continue sending the batches which are in
// retry for partitions that were successfully added.
removeAllPartitionsOnFatalOrAbortedError := func() {
t.pendingPartitionsInCurrentTxn = topicPartitionSet{}
}
// We only want to reduce the backoff when retrying the first AddPartition which errored out due to a
// CONCURRENT_TRANSACTIONS error since this means that the previous transaction is still completing and
// we don't want to wait too long before trying to start the new one.
//
// This is only a temporary fix, the long term solution is being tracked in
// https://issues.apache.org/jira/browse/KAFKA-5482
retryBackoff := t.client.Config().Producer.Transaction.Retry.Backoff
computeBackoff := func(attemptsRemaining int) time.Duration {
if t.client.Config().Producer.Transaction.Retry.BackoffFunc != nil {
maxRetries := t.client.Config().Producer.Transaction.Retry.Max
retries := maxRetries - attemptsRemaining
return t.client.Config().Producer.Transaction.Retry.BackoffFunc(retries, maxRetries)
}
return retryBackoff
}
attemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max
exec := func(run func() (bool, error), err error) error {
for attemptsRemaining >= 0 {
var retry bool
retry, err = run()
if !retry {
return err
}
backoff := computeBackoff(attemptsRemaining)
Logger.Printf("txnmgr/add-partition-to-txn retrying after %dms... (%d attempts remaining) (%s)\n", backoff/time.Millisecond, attemptsRemaining, err)
time.Sleep(backoff)
attemptsRemaining--
}
return err
}
return exec(func() (bool, error) {
coordinator, err := t.client.TransactionCoordinator(t.transactionalID)
if err != nil {
return true, err
}
request := &AddPartitionsToTxnRequest{
TransactionalID: t.transactionalID,
ProducerID: t.producerID,
ProducerEpoch: t.producerEpoch,
TopicPartitions: t.pendingPartitionsInCurrentTxn.mapToRequest(),
}
if t.client.Config().Version.IsAtLeast(V2_7_0_0) {
// Version 2 adds the support for new error code PRODUCER_FENCED.
request.Version = 2
} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {
// Version 1 is the same as version 0.
request.Version = 1
}
addPartResponse, err := coordinator.AddPartitionsToTxn(request)
if err != nil {
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
return true, err
}
if addPartResponse == nil {
return true, ErrTxnUnableToParseResponse
}
// remove from the list partitions that have been successfully updated
var responseErrors []error
for topic, results := range addPartResponse.Errors {
for _, response := range results {
tp := topicPartition{topic: topic, partition: response.Partition}
switch response.Err {
case ErrNoError:
// Mark partition as added to transaction
t.partitionsInCurrentTxn[tp] = struct{}{}
delete(t.pendingPartitionsInCurrentTxn, tp)
continue
case ErrConsumerCoordinatorNotAvailable:
fallthrough
case ErrNotCoordinatorForConsumer:
_ = coordinator.Close()
_ = t.client.RefreshTransactionCoordinator(t.transactionalID)
fallthrough
case ErrUnknownTopicOrPartition:
fallthrough
case ErrOffsetsLoadInProgress:
// Retry topicPartition
case ErrConcurrentTransactions:
if len(t.partitionsInCurrentTxn) == 0 && retryBackoff > addPartitionsRetryBackoff {
retryBackoff = addPartitionsRetryBackoff
}
case ErrOperationNotAttempted:
fallthrough
case ErrTopicAuthorizationFailed:
removeAllPartitionsOnFatalOrAbortedError()
return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, response.Err)
case ErrUnknownProducerID:
fallthrough
case ErrInvalidProducerIDMapping:
removeAllPartitionsOnFatalOrAbortedError()
return false, t.abortableErrorIfPossible(response.Err)
// Fatal errors
default:
removeAllPartitionsOnFatalOrAbortedError()
return false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)
}
responseErrors = append(responseErrors, response.Err)
}
}
// handle end
if len(t.pendingPartitionsInCurrentTxn) == 0 {
DebugLogger.Printf("txnmgr/add-partition-to-txn [%s] successful to add partitions txn %+v\n",
t.transactionalID, addPartResponse)
return false, nil
}
return true, Wrap(ErrAddPartitionsToTxn, responseErrors...)
}, nil)
}
// Build a new transaction manager sharing producer client.
func newTransactionManager(conf *Config, client Client) (*transactionManager, error) {
txnmgr := &transactionManager{
producerID: noProducerID,
producerEpoch: noProducerEpoch,
client: client,
pendingPartitionsInCurrentTxn: topicPartitionSet{},
partitionsInCurrentTxn: topicPartitionSet{},
offsetsInCurrentTxn: make(map[string]topicPartitionOffsets),
status: ProducerTxnFlagUninitialized,
}
if conf.Producer.Idempotent {
txnmgr.transactionalID = conf.Producer.Transaction.ID
txnmgr.transactionTimeout = conf.Producer.Transaction.Timeout
txnmgr.sequenceNumbers = make(map[string]int32)
txnmgr.mutex = sync.Mutex{}
var err error
txnmgr.producerID, txnmgr.producerEpoch, err = txnmgr.initProducerId()
if err != nil {
return nil, err
}
Logger.Printf("txnmgr/init-producer-id [%s] obtained a ProducerId: %d and ProducerEpoch: %d\n",
txnmgr.transactionalID, txnmgr.producerID, txnmgr.producerEpoch)
}
return txnmgr, nil
}
// re-init producer-id and producer-epoch if needed.
func (t *transactionManager) initializeTransactions() (err error) {
t.producerID, t.producerEpoch, err = t.initProducerId()
return
}
golang-github-ibm-sarama-1.43.2/transaction_manager_test.go 0000664 0000000 0000000 00000073763 14612567413 0023775 0 ustar 00root root 0000000 0000000 package sarama
import (
"errors"
"testing"
"github.com/stretchr/testify/require"
)
func TestTransitions(t *testing.T) {
testError := errors.New("test")
type testCase struct {
transitions []ProducerTxnStatusFlag
expectedError error
}
testCases := []testCase{
{
transitions: []ProducerTxnStatusFlag{
ProducerTxnFlagUninitialized,
ProducerTxnFlagReady,
ProducerTxnFlagInTransaction,
ProducerTxnFlagEndTransaction | ProducerTxnFlagCommittingTransaction,
ProducerTxnFlagReady,
},
expectedError: nil,
},
{
transitions: []ProducerTxnStatusFlag{
ProducerTxnFlagUninitialized,
ProducerTxnFlagReady,
ProducerTxnFlagInTransaction,
ProducerTxnFlagEndTransaction | ProducerTxnFlagAbortingTransaction,
ProducerTxnFlagReady,
},
expectedError: nil,
},
{
transitions: []ProducerTxnStatusFlag{
ProducerTxnFlagUninitialized,
ProducerTxnFlagReady,
ProducerTxnFlagInTransaction,
ProducerTxnFlagEndTransaction,
ProducerTxnFlagInError | ProducerTxnFlagAbortableError,
},
expectedError: testError,
},
{
transitions: []ProducerTxnStatusFlag{
ProducerTxnFlagInError | ProducerTxnFlagAbortableError,
ProducerTxnFlagEndTransaction | ProducerTxnFlagAbortingTransaction,
ProducerTxnFlagReady,
},
expectedError: nil,
},
{
transitions: []ProducerTxnStatusFlag{
ProducerTxnFlagInError | ProducerTxnFlagAbortableError,
ProducerTxnFlagEndTransaction | ProducerTxnFlagCommittingTransaction,
},
expectedError: ErrTransitionNotAllowed,
},
{
transitions: []ProducerTxnStatusFlag{
ProducerTxnFlagInError | ProducerTxnFlagFatalError,
ProducerTxnFlagEndTransaction | ProducerTxnFlagAbortingTransaction,
},
expectedError: ErrTransitionNotAllowed,
},
}
for _, tc := range testCases {
txnmgr := transactionManager{}
txnmgr.status = tc.transitions[0]
var lastError error
for i := 1; i < len(tc.transitions); i++ {
var baseErr error
if tc.transitions[i]&ProducerTxnFlagInError != 0 {
baseErr = testError
}
lastError = txnmgr.transitionTo(tc.transitions[i], baseErr)
}
require.Equal(t, tc.expectedError, lastError, tc)
}
}
func TestTxnmgrInitProducerIdTxn(t *testing.T) {
broker := NewMockBroker(t, 1)
defer broker.Close()
metadataLeader := new(MetadataResponse)
metadataLeader.Version = 4
metadataLeader.ControllerID = broker.brokerID
metadataLeader.AddBroker(broker.Addr(), broker.BrokerID())
broker.Returns(metadataLeader)
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "test"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
findCoordinatorResponse := FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
}
broker.Returns(&findCoordinatorResponse)
producerIdResponse := &InitProducerIDResponse{
Err: ErrNoError,
ProducerID: 1,
ProducerEpoch: 0,
}
broker.Returns(producerIdResponse)
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
require.Equal(t, int64(1), txmng.producerID)
require.Equal(t, int16(0), txmng.producerEpoch)
require.Equal(t, ProducerTxnFlagReady, txmng.status)
}
// TestTxnmgrInitProducerIdTxnCoordinatorLoading ensure we retry initProducerId when either FindCoordinator or InitProducerID returns ErrOffsetsLoadInProgress
func TestTxnmgrInitProducerIdTxnCoordinatorLoading(t *testing.T) {
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "txid-group"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
broker := NewMockBroker(t, 1)
defer broker.Close()
broker.SetHandlerByMap(map[string]MockResponse{
"MetadataRequest": NewMockMetadataResponse(t).
SetController(broker.BrokerID()).
SetBroker(broker.Addr(), broker.BrokerID()),
"FindCoordinatorRequest": NewMockSequence(
NewMockFindCoordinatorResponse(t).
SetError(CoordinatorTransaction, "txid-group", ErrOffsetsLoadInProgress),
NewMockFindCoordinatorResponse(t).
SetError(CoordinatorTransaction, "txid-group", ErrOffsetsLoadInProgress),
NewMockFindCoordinatorResponse(t).
SetCoordinator(CoordinatorTransaction, "txid-group", broker),
),
"InitProducerIDRequest": NewMockSequence(
NewMockInitProducerIDResponse(t).
SetError(ErrOffsetsLoadInProgress),
NewMockInitProducerIDResponse(t).
SetError(ErrOffsetsLoadInProgress),
NewMockInitProducerIDResponse(t).
SetProducerID(1).
SetProducerEpoch(0),
),
})
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
require.Equal(t, int64(1), txmng.producerID)
require.Equal(t, int16(0), txmng.producerEpoch)
require.Equal(t, ProducerTxnFlagReady, txmng.status)
}
func TestMaybeAddPartitionToCurrentTxn(t *testing.T) {
type testCase struct {
initialFlags ProducerTxnStatusFlag
initialPartitionsInCurrentTxn topicPartitionSet
initialPendingPartitionsInCurrentTxn topicPartitionSet
tpToAdd map[string][]int32
expectedPendingPartitions topicPartitionSet
expectedPartitionsInTxn topicPartitionSet
}
testCases := []testCase{
{
initialFlags: ProducerTxnFlagInTransaction,
initialPartitionsInCurrentTxn: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
initialPendingPartitionsInCurrentTxn: topicPartitionSet{},
tpToAdd: map[string][]int32{
"test-topic": {
0,
},
},
expectedPendingPartitions: topicPartitionSet{},
expectedPartitionsInTxn: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
},
{
initialFlags: ProducerTxnFlagInTransaction,
initialPartitionsInCurrentTxn: topicPartitionSet{},
initialPendingPartitionsInCurrentTxn: topicPartitionSet{},
tpToAdd: map[string][]int32{
"test-topic": {
0,
},
},
expectedPendingPartitions: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
expectedPartitionsInTxn: topicPartitionSet{},
},
{
initialFlags: ProducerTxnFlagInTransaction,
initialPartitionsInCurrentTxn: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
initialPendingPartitionsInCurrentTxn: topicPartitionSet{},
tpToAdd: map[string][]int32{
"test-topic": {
0,
},
},
expectedPendingPartitions: topicPartitionSet{},
expectedPartitionsInTxn: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
},
{
initialFlags: ProducerTxnFlagInTransaction,
initialPartitionsInCurrentTxn: topicPartitionSet{},
initialPendingPartitionsInCurrentTxn: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
tpToAdd: map[string][]int32{
"test-topic": {
0,
},
},
expectedPendingPartitions: topicPartitionSet{
{topic: "test-topic", partition: 0}: struct{}{},
},
expectedPartitionsInTxn: topicPartitionSet{},
},
{
initialFlags: ProducerTxnFlagInError,
initialPartitionsInCurrentTxn: topicPartitionSet{},
initialPendingPartitionsInCurrentTxn: topicPartitionSet{},
tpToAdd: map[string][]int32{
"test-topic": {
0,
},
},
expectedPendingPartitions: topicPartitionSet{},
expectedPartitionsInTxn: topicPartitionSet{},
},
}
broker := NewMockBroker(t, 1)
defer broker.Close()
metadataLeader := new(MetadataResponse)
metadataLeader.Version = 4
metadataLeader.ControllerID = broker.brokerID
metadataLeader.AddBroker(broker.Addr(), broker.BrokerID())
metadataLeader.AddTopic("test-topic", ErrNoError)
metadataLeader.AddTopicPartition("test-topic", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "test"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
config.Producer.Transaction.Retry.Max = 0
config.Producer.Transaction.Retry.Backoff = 0
for _, tc := range testCases {
func() {
broker.Returns(metadataLeader)
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
findCoordinatorResponse := FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
}
broker.Returns(&findCoordinatorResponse)
producerIdResponse := &InitProducerIDResponse{
Err: ErrNoError,
ProducerID: 1,
ProducerEpoch: 0,
}
broker.Returns(producerIdResponse)
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
txmng.partitionsInCurrentTxn = tc.initialPartitionsInCurrentTxn
txmng.pendingPartitionsInCurrentTxn = tc.initialPendingPartitionsInCurrentTxn
txmng.status = tc.initialFlags
for topic, partitions := range tc.tpToAdd {
for _, partition := range partitions {
txmng.maybeAddPartitionToCurrentTxn(topic, partition)
}
}
require.Equal(t, tc.expectedPartitionsInTxn, txmng.partitionsInCurrentTxn, tc)
require.Equal(t, tc.expectedPendingPartitions, txmng.pendingPartitionsInCurrentTxn, tc)
}()
}
}
func TestAddOffsetsToTxn(t *testing.T) {
type testCase struct {
brokerErr KError
initialFlags ProducerTxnStatusFlag
expectedFlags ProducerTxnStatusFlag
expectedError error
newOffsets topicPartitionOffsets
}
originalOffsets := topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
}
testCases := []testCase{
{
brokerErr: ErrNoError,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: nil,
newOffsets: topicPartitionOffsets{},
},
{
brokerErr: ErrConsumerCoordinatorNotAvailable,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: ErrConsumerCoordinatorNotAvailable,
newOffsets: originalOffsets,
},
{
brokerErr: ErrNotCoordinatorForConsumer,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: ErrNotCoordinatorForConsumer,
newOffsets: originalOffsets,
},
{
brokerErr: ErrOffsetsLoadInProgress,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: ErrOffsetsLoadInProgress,
newOffsets: originalOffsets,
},
{
brokerErr: ErrConcurrentTransactions,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: ErrConcurrentTransactions,
newOffsets: originalOffsets,
},
{
brokerErr: ErrUnknownProducerID,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrUnknownProducerID,
newOffsets: originalOffsets,
},
{
brokerErr: ErrInvalidProducerIDMapping,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrInvalidProducerIDMapping,
newOffsets: originalOffsets,
},
{
brokerErr: ErrGroupAuthorizationFailed,
initialFlags: ProducerTxnFlagInTransaction,
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrGroupAuthorizationFailed,
newOffsets: originalOffsets,
},
}
broker := NewMockBroker(t, 1)
defer broker.Close()
metadataLeader := new(MetadataResponse)
metadataLeader.Version = 4
metadataLeader.ControllerID = broker.brokerID
metadataLeader.AddBroker(broker.Addr(), broker.BrokerID())
metadataLeader.AddTopic("test-topic", ErrNoError)
metadataLeader.AddTopicPartition("test-topic", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "test"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
config.Producer.Transaction.Retry.Max = 0
config.Producer.Transaction.Retry.Backoff = 0
offsets := topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
}
for _, tc := range testCases {
func() {
broker.Returns(metadataLeader)
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
findCoordinatorResponse := FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
}
broker.Returns(&findCoordinatorResponse)
producerIdResponse := &InitProducerIDResponse{
Err: ErrNoError,
ProducerID: 1,
ProducerEpoch: 0,
}
broker.Returns(producerIdResponse)
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
txmng.status = tc.initialFlags
broker.Returns(&AddOffsetsToTxnResponse{
Err: tc.brokerErr,
})
if errors.Is(tc.brokerErr, ErrRequestTimedOut) ||
errors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||
errors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {
broker.Returns(&FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
})
}
if tc.brokerErr == ErrNoError {
broker.Returns(&FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
})
broker.Returns(&TxnOffsetCommitResponse{
Topics: map[string][]*PartitionError{
"test-topic": {
{
Partition: 0,
Err: ErrNoError,
},
},
},
})
}
newOffsets, err := txmng.publishOffsetsToTxn(offsets, "test-group")
if tc.expectedError != nil {
require.Equal(t, tc.expectedError.Error(), err.Error())
} else {
require.Equal(t, tc.expectedError, err)
}
require.Equal(t, tc.newOffsets, newOffsets)
require.True(t, tc.expectedFlags&txmng.status != 0)
}()
}
}
func TestTxnOffsetsCommit(t *testing.T) {
type testCase struct {
brokerErr KError
initialFlags ProducerTxnStatusFlag
initialOffsets topicPartitionOffsets
expectedFlags ProducerTxnStatusFlag
expectedError error
expectedOffsets topicPartitionOffsets
}
originalOffsets := topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
}
testCases := []testCase{
{
brokerErr: ErrConsumerCoordinatorNotAvailable,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrTxnOffsetCommit, ErrConsumerCoordinatorNotAvailable),
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrNotCoordinatorForConsumer,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrTxnOffsetCommit, ErrNotCoordinatorForConsumer),
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrNoError,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: nil,
expectedOffsets: topicPartitionOffsets{},
},
{
brokerErr: ErrUnknownTopicOrPartition,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrTxnOffsetCommit, ErrUnknownTopicOrPartition),
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrOffsetsLoadInProgress,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrTxnOffsetCommit, ErrOffsetsLoadInProgress),
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrIllegalGeneration,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrIllegalGeneration,
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrUnknownMemberId,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrUnknownMemberId,
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrFencedInstancedId,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrFencedInstancedId,
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrGroupAuthorizationFailed,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrGroupAuthorizationFailed,
expectedOffsets: originalOffsets,
},
{
brokerErr: ErrKafkaStorageError,
initialFlags: ProducerTxnFlagInTransaction,
initialOffsets: topicPartitionOffsets{
topicPartition{topic: "test-topic", partition: 0}: {
Partition: 0,
Offset: 0,
},
},
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrKafkaStorageError,
expectedOffsets: originalOffsets,
},
}
broker := NewMockBroker(t, 1)
defer broker.Close()
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "test"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
config.Producer.Transaction.Retry.Max = 0
config.Producer.Transaction.Retry.Backoff = 0
metadataLeader := new(MetadataResponse)
metadataLeader.Version = 4
metadataLeader.ControllerID = broker.brokerID
metadataLeader.AddBroker(broker.Addr(), broker.BrokerID())
metadataLeader.AddTopic("test-topic", ErrNoError)
metadataLeader.AddTopicPartition("test-topic", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)
for _, tc := range testCases {
func() {
broker.Returns(metadataLeader)
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
findCoordinatorResponse := FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
}
broker.Returns(&findCoordinatorResponse)
producerIdResponse := &InitProducerIDResponse{
Err: ErrNoError,
ProducerID: 1,
ProducerEpoch: 0,
}
broker.Returns(producerIdResponse)
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
txmng.status = tc.initialFlags
broker.Returns(&AddOffsetsToTxnResponse{
Err: ErrNoError,
})
broker.Returns(&FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
})
broker.Returns(&TxnOffsetCommitResponse{
Topics: map[string][]*PartitionError{
"test-topic": {
{
Partition: 0,
Err: tc.brokerErr,
},
},
},
})
if errors.Is(tc.brokerErr, ErrRequestTimedOut) ||
errors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||
errors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {
broker.Returns(&FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
})
}
newOffsets, err := txmng.publishOffsetsToTxn(tc.initialOffsets, "test-group")
if tc.expectedError != nil {
require.Equal(t, tc.expectedError.Error(), err.Error())
} else {
require.Equal(t, tc.expectedError, err)
}
require.Equal(t, tc.expectedOffsets, newOffsets)
require.True(t, tc.expectedFlags&txmng.status != 0)
}()
}
}
func TestEndTxn(t *testing.T) {
type testCase struct {
brokerErr KError
commit bool
expectedFlags ProducerTxnStatusFlag
expectedError error
}
testCases := []testCase{
{
brokerErr: ErrNoError,
commit: true,
expectedFlags: ProducerTxnFlagReady,
expectedError: nil,
},
{
brokerErr: ErrConsumerCoordinatorNotAvailable,
commit: true,
expectedFlags: ProducerTxnFlagEndTransaction,
expectedError: ErrConsumerCoordinatorNotAvailable,
},
{
brokerErr: ErrNotCoordinatorForConsumer,
commit: true,
expectedFlags: ProducerTxnFlagEndTransaction,
expectedError: ErrNotCoordinatorForConsumer,
},
{
brokerErr: ErrOffsetsLoadInProgress,
commit: true,
expectedFlags: ProducerTxnFlagEndTransaction,
expectedError: ErrOffsetsLoadInProgress,
},
{
brokerErr: ErrConcurrentTransactions,
commit: true,
expectedFlags: ProducerTxnFlagEndTransaction,
expectedError: ErrConcurrentTransactions,
},
{
brokerErr: ErrUnknownProducerID,
commit: true,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrUnknownProducerID,
},
{
brokerErr: ErrInvalidProducerIDMapping,
commit: true,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrInvalidProducerIDMapping,
},
}
broker := NewMockBroker(t, 1)
defer broker.Close()
metadataLeader := new(MetadataResponse)
metadataLeader.Version = 4
metadataLeader.ControllerID = broker.brokerID
metadataLeader.AddBroker(broker.Addr(), broker.BrokerID())
metadataLeader.AddTopic("test-topic", ErrNoError)
metadataLeader.AddTopicPartition("test-topic", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "test"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
config.Producer.Transaction.Retry.Max = 0
config.Producer.Transaction.Retry.Backoff = 0
for _, tc := range testCases {
func() {
broker.Returns(metadataLeader)
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
findCoordinatorResponse := FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
}
broker.Returns(&findCoordinatorResponse)
producerIdResponse := &InitProducerIDResponse{
Err: ErrNoError,
ProducerID: 1,
ProducerEpoch: 0,
}
broker.Returns(producerIdResponse)
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
txmng.status = ProducerTxnFlagEndTransaction
endTxnResponse := &EndTxnResponse{
Err: tc.brokerErr,
ThrottleTime: 0,
}
broker.Returns(endTxnResponse)
if errors.Is(tc.brokerErr, ErrRequestTimedOut) ||
errors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||
errors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {
broker.Returns(&FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
})
}
err = txmng.endTxn(tc.commit)
require.Equal(t, tc.expectedError, err)
require.True(t, txmng.currentTxnStatus()&tc.expectedFlags != 0)
}()
}
}
func TestPublishPartitionToTxn(t *testing.T) {
type testCase struct {
brokerErr KError
expectedFlags ProducerTxnStatusFlag
expectedError error
expectedPendingPartitions topicPartitionSet
expectedPartitionsInTxn topicPartitionSet
}
initialPendingTopicPartitionSet := topicPartitionSet{
{
topic: "test-topic",
partition: 0,
}: struct{}{},
}
testCases := []testCase{
{
brokerErr: ErrNoError,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: nil,
expectedPendingPartitions: topicPartitionSet{},
expectedPartitionsInTxn: initialPendingTopicPartitionSet,
},
{
brokerErr: ErrConsumerCoordinatorNotAvailable,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrAddPartitionsToTxn, ErrConsumerCoordinatorNotAvailable),
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: initialPendingTopicPartitionSet,
},
{
brokerErr: ErrNotCoordinatorForConsumer,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrAddPartitionsToTxn, ErrNotCoordinatorForConsumer),
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: initialPendingTopicPartitionSet,
},
{
brokerErr: ErrUnknownTopicOrPartition,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrAddPartitionsToTxn, ErrUnknownTopicOrPartition),
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: initialPendingTopicPartitionSet,
},
{
brokerErr: ErrOffsetsLoadInProgress,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrAddPartitionsToTxn, ErrOffsetsLoadInProgress),
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: initialPendingTopicPartitionSet,
},
{
brokerErr: ErrConcurrentTransactions,
expectedFlags: ProducerTxnFlagInTransaction,
expectedError: Wrap(ErrAddPartitionsToTxn, ErrConcurrentTransactions),
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: initialPendingTopicPartitionSet,
},
{
brokerErr: ErrOperationNotAttempted,
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrOperationNotAttempted,
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: topicPartitionSet{},
},
{
brokerErr: ErrTopicAuthorizationFailed,
expectedFlags: ProducerTxnFlagAbortableError,
expectedError: ErrTopicAuthorizationFailed,
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: topicPartitionSet{},
},
{
brokerErr: ErrUnknownProducerID,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrUnknownProducerID,
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: topicPartitionSet{},
},
{
brokerErr: ErrInvalidProducerIDMapping,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrInvalidProducerIDMapping,
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: topicPartitionSet{},
},
{
brokerErr: ErrKafkaStorageError,
expectedFlags: ProducerTxnFlagFatalError,
expectedError: ErrKafkaStorageError,
expectedPartitionsInTxn: topicPartitionSet{},
expectedPendingPartitions: topicPartitionSet{},
},
}
broker := NewMockBroker(t, 1)
defer broker.Close()
metadataLeader := new(MetadataResponse)
metadataLeader.Version = 4
metadataLeader.ControllerID = broker.brokerID
metadataLeader.AddBroker(broker.Addr(), broker.BrokerID())
metadataLeader.AddTopic("test-topic", ErrNoError)
metadataLeader.AddTopicPartition("test-topic", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)
config := NewTestConfig()
config.Producer.Idempotent = true
config.Producer.Transaction.ID = "test"
config.Version = V0_11_0_0
config.Producer.RequiredAcks = WaitForAll
config.Net.MaxOpenRequests = 1
config.Producer.Transaction.Retry.Max = 0
config.Producer.Transaction.Retry.Backoff = 0
for _, tc := range testCases {
func() {
broker.Returns(metadataLeader)
client, err := NewClient([]string{broker.Addr()}, config)
require.NoError(t, err)
defer client.Close()
findCoordinatorResponse := FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
}
broker.Returns(&findCoordinatorResponse)
producerIdResponse := &InitProducerIDResponse{
Err: ErrNoError,
ProducerID: 1,
ProducerEpoch: 0,
}
broker.Returns(producerIdResponse)
txmng, err := newTransactionManager(config, client)
require.NoError(t, err)
txmng.status = ProducerTxnFlagInTransaction
txmng.pendingPartitionsInCurrentTxn = topicPartitionSet{
{
topic: "test-topic",
partition: 0,
}: struct{}{},
}
broker.Returns(&AddPartitionsToTxnResponse{
Errors: map[string][]*PartitionError{
"test-topic": {
{
Partition: 0,
Err: tc.brokerErr,
},
},
},
})
if errors.Is(tc.brokerErr, ErrRequestTimedOut) ||
errors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||
errors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {
broker.Returns(&FindCoordinatorResponse{
Coordinator: client.Brokers()[0],
Err: ErrNoError,
Version: 1,
})
}
err = txmng.publishTxnPartitions()
if tc.expectedError != nil {
require.Equal(t, tc.expectedError.Error(), err.Error(), tc)
} else {
require.Equal(t, tc.expectedError, err, tc)
}
require.True(t, txmng.status&tc.expectedFlags != 0, tc)
require.Equal(t, tc.expectedPartitionsInTxn, txmng.partitionsInCurrentTxn, tc)
require.Equal(t, tc.expectedPendingPartitions, txmng.pendingPartitionsInCurrentTxn, tc)
}()
}
}
golang-github-ibm-sarama-1.43.2/txn_offset_commit_request.go 0000664 0000000 0000000 00000006547 14612567413 0024212 0 ustar 00root root 0000000 0000000 package sarama
type TxnOffsetCommitRequest struct {
Version int16
TransactionalID string
GroupID string
ProducerID int64
ProducerEpoch int16
Topics map[string][]*PartitionOffsetMetadata
}
func (t *TxnOffsetCommitRequest) encode(pe packetEncoder) error {
if err := pe.putString(t.TransactionalID); err != nil {
return err
}
if err := pe.putString(t.GroupID); err != nil {
return err
}
pe.putInt64(t.ProducerID)
pe.putInt16(t.ProducerEpoch)
if err := pe.putArrayLength(len(t.Topics)); err != nil {
return err
}
for topic, partitions := range t.Topics {
if err := pe.putString(topic); err != nil {
return err
}
if err := pe.putArrayLength(len(partitions)); err != nil {
return err
}
for _, partition := range partitions {
if err := partition.encode(pe, t.Version); err != nil {
return err
}
}
}
return nil
}
func (t *TxnOffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) {
t.Version = version
if t.TransactionalID, err = pd.getString(); err != nil {
return err
}
if t.GroupID, err = pd.getString(); err != nil {
return err
}
if t.ProducerID, err = pd.getInt64(); err != nil {
return err
}
if t.ProducerEpoch, err = pd.getInt16(); err != nil {
return err
}
n, err := pd.getArrayLength()
if err != nil {
return err
}
t.Topics = make(map[string][]*PartitionOffsetMetadata)
for i := 0; i < n; i++ {
topic, err := pd.getString()
if err != nil {
return err
}
m, err := pd.getArrayLength()
if err != nil {
return err
}
t.Topics[topic] = make([]*PartitionOffsetMetadata, m)
for j := 0; j < m; j++ {
partitionOffsetMetadata := new(PartitionOffsetMetadata)
if err := partitionOffsetMetadata.decode(pd, version); err != nil {
return err
}
t.Topics[topic][j] = partitionOffsetMetadata
}
}
return nil
}
func (a *TxnOffsetCommitRequest) key() int16 {
return 28
}
func (a *TxnOffsetCommitRequest) version() int16 {
return a.Version
}
func (a *TxnOffsetCommitRequest) headerVersion() int16 {
return 1
}
func (a *TxnOffsetCommitRequest) isValidVersion() bool {
return a.Version >= 0 && a.Version <= 2
}
func (a *TxnOffsetCommitRequest) requiredVersion() KafkaVersion {
switch a.Version {
case 2:
return V2_1_0_0
case 1:
return V2_0_0_0
case 0:
return V0_11_0_0
default:
return V2_1_0_0
}
}
type PartitionOffsetMetadata struct {
// Partition contains the index of the partition within the topic.
Partition int32
// Offset contains the message offset to be committed.
Offset int64
// LeaderEpoch contains the leader epoch of the last consumed record.
LeaderEpoch int32
// Metadata contains any associated metadata the client wants to keep.
Metadata *string
}
func (p *PartitionOffsetMetadata) encode(pe packetEncoder, version int16) error {
pe.putInt32(p.Partition)
pe.putInt64(p.Offset)
if version >= 2 {
pe.putInt32(p.LeaderEpoch)
}
if err := pe.putNullableString(p.Metadata); err != nil {
return err
}
return nil
}
func (p *PartitionOffsetMetadata) decode(pd packetDecoder, version int16) (err error) {
if p.Partition, err = pd.getInt32(); err != nil {
return err
}
if p.Offset, err = pd.getInt64(); err != nil {
return err
}
if version >= 2 {
if p.LeaderEpoch, err = pd.getInt32(); err != nil {
return err
}
}
if p.Metadata, err = pd.getNullableString(); err != nil {
return err
}
return nil
}
golang-github-ibm-sarama-1.43.2/txn_offset_commit_request_test.go 0000664 0000000 0000000 00000002773 14612567413 0025246 0 ustar 00root root 0000000 0000000 package sarama
import "testing"
var (
txnOffsetCommitRequest = []byte{
0, 3, 't', 'x', 'n',
0, 7, 'g', 'r', 'o', 'u', 'p', 'i', 'd',
0, 0, 0, 0, 0, 0, 31, 64, // producer ID
0, 1, // producer epoch
0, 0, 0, 1, // 1 topic
0, 5, 't', 'o', 'p', 'i', 'c',
0, 0, 0, 1, // 1 partition
0, 0, 0, 2, // partition no 2
0, 0, 0, 0, 0, 0, 0, 123,
255, 255, // no meta data
}
txnOffsetCommitRequestV2 = []byte{
0, 3, 't', 'x', 'n',
0, 7, 'g', 'r', 'o', 'u', 'p', 'i', 'd',
0, 0, 0, 0, 0, 0, 31, 64, // producer ID
0, 1, // producer epoch
0, 0, 0, 1, // 1 topic
0, 5, 't', 'o', 'p', 'i', 'c',
0, 0, 0, 1, // 1 partition
0, 0, 0, 2, // partition no 2
0, 0, 0, 0, 0, 0, 0, 123,
0, 0, 0, 9, // leader epoch
255, 255, // no meta data
}
)
func TestTxnOffsetCommitRequest(t *testing.T) {
req := &TxnOffsetCommitRequest{
TransactionalID: "txn",
GroupID: "groupid",
ProducerID: 8000,
ProducerEpoch: 1,
Topics: map[string][]*PartitionOffsetMetadata{
"topic": {{
Offset: 123,
Partition: 2,
}},
},
}
testRequest(t, "V0", req, txnOffsetCommitRequest)
}
func TestTxnOffsetCommitRequestV2(t *testing.T) {
req := &TxnOffsetCommitRequest{
Version: 2,
TransactionalID: "txn",
GroupID: "groupid",
ProducerID: 8000,
ProducerEpoch: 1,
Topics: map[string][]*PartitionOffsetMetadata{
"topic": {{
Offset: 123,
Partition: 2,
LeaderEpoch: 9,
}},
},
}
testRequest(t, "V2", req, txnOffsetCommitRequestV2)
}
golang-github-ibm-sarama-1.43.2/txn_offset_commit_response.go 0000664 0000000 0000000 00000003737 14612567413 0024356 0 ustar 00root root 0000000 0000000 package sarama
import (
"time"
)
type TxnOffsetCommitResponse struct {
Version int16
ThrottleTime time.Duration
Topics map[string][]*PartitionError
}
func (t *TxnOffsetCommitResponse) encode(pe packetEncoder) error {
pe.putInt32(int32(t.ThrottleTime / time.Millisecond))
if err := pe.putArrayLength(len(t.Topics)); err != nil {
return err
}
for topic, e := range t.Topics {
if err := pe.putString(topic); err != nil {
return err
}
if err := pe.putArrayLength(len(e)); err != nil {
return err
}
for _, partitionError := range e {
if err := partitionError.encode(pe); err != nil {
return err
}
}
}
return nil
}
func (t *TxnOffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) {
t.Version = version
throttleTime, err := pd.getInt32()
if err != nil {
return err
}
t.ThrottleTime = time.Duration(throttleTime) * time.Millisecond
n, err := pd.getArrayLength()
if err != nil {
return err
}
t.Topics = make(map[string][]*PartitionError)
for i := 0; i < n; i++ {
topic, err := pd.getString()
if err != nil {
return err
}
m, err := pd.getArrayLength()
if err != nil {
return err
}
t.Topics[topic] = make([]*PartitionError, m)
for j := 0; j < m; j++ {
t.Topics[topic][j] = new(PartitionError)
if err := t.Topics[topic][j].decode(pd, version); err != nil {
return err
}
}
}
return nil
}
func (a *TxnOffsetCommitResponse) key() int16 {
return 28
}
func (a *TxnOffsetCommitResponse) version() int16 {
return a.Version
}
func (a *TxnOffsetCommitResponse) headerVersion() int16 {
return 0
}
func (a *TxnOffsetCommitResponse) isValidVersion() bool {
return a.Version >= 0 && a.Version <= 2
}
func (a *TxnOffsetCommitResponse) requiredVersion() KafkaVersion {
switch a.Version {
case 2:
return V2_1_0_0
case 1:
return V2_0_0_0
case 0:
return V0_11_0_0
default:
return V2_1_0_0
}
}
func (r *TxnOffsetCommitResponse) throttleTime() time.Duration {
return r.ThrottleTime
}
golang-github-ibm-sarama-1.43.2/txn_offset_commit_response_test.go 0000664 0000000 0000000 00000001051 14612567413 0025400 0 ustar 00root root 0000000 0000000 package sarama
import (
"testing"
"time"
)
var txnOffsetCommitResponse = []byte{
0, 0, 0, 100,
0, 0, 0, 1, // 1 topic
0, 5, 't', 'o', 'p', 'i', 'c',
0, 0, 0, 1, // 1 partition response
0, 0, 0, 2, // partition number 2
0, 47, // err
}
func TestTxnOffsetCommitResponse(t *testing.T) {
resp := &TxnOffsetCommitResponse{
ThrottleTime: 100 * time.Millisecond,
Topics: map[string][]*PartitionError{
"topic": {{
Partition: 2,
Err: ErrInvalidProducerEpoch,
}},
},
}
testResponse(t, "", resp, txnOffsetCommitResponse)
}
golang-github-ibm-sarama-1.43.2/utils.go 0000664 0000000 0000000 00000020105 14612567413 0020035 0 ustar 00root root 0000000 0000000 package sarama
import (
"bufio"
"fmt"
"net"
"regexp"
)
type none struct{}
// make []int32 sortable so we can sort partition numbers
type int32Slice []int32
func (slice int32Slice) Len() int {
return len(slice)
}
func (slice int32Slice) Less(i, j int) bool {
return slice[i] < slice[j]
}
func (slice int32Slice) Swap(i, j int) {
slice[i], slice[j] = slice[j], slice[i]
}
func dupInt32Slice(input []int32) []int32 {
ret := make([]int32, 0, len(input))
ret = append(ret, input...)
return ret
}
func withRecover(fn func()) {
defer func() {
handler := PanicHandler
if handler != nil {
if err := recover(); err != nil {
handler(err)
}
}
}()
fn()
}
func safeAsyncClose(b *Broker) {
tmp := b // local var prevents clobbering in goroutine
go withRecover(func() {
if connected, _ := tmp.Connected(); connected {
if err := tmp.Close(); err != nil {
Logger.Println("Error closing broker", tmp.ID(), ":", err)
}
}
})
}
// Encoder is a simple interface for any type that can be encoded as an array of bytes
// in order to be sent as the key or value of a Kafka message. Length() is provided as an
// optimization, and must return the same as len() on the result of Encode().
type Encoder interface {
Encode() ([]byte, error)
Length() int
}
// make strings and byte slices encodable for convenience so they can be used as keys
// and/or values in kafka messages
// StringEncoder implements the Encoder interface for Go strings so that they can be used
// as the Key or Value in a ProducerMessage.
type StringEncoder string
func (s StringEncoder) Encode() ([]byte, error) {
return []byte(s), nil
}
func (s StringEncoder) Length() int {
return len(s)
}
// ByteEncoder implements the Encoder interface for Go byte slices so that they can be used
// as the Key or Value in a ProducerMessage.
type ByteEncoder []byte
func (b ByteEncoder) Encode() ([]byte, error) {
return b, nil
}
func (b ByteEncoder) Length() int {
return len(b)
}
// bufConn wraps a net.Conn with a buffer for reads to reduce the number of
// reads that trigger syscalls.
type bufConn struct {
net.Conn
buf *bufio.Reader
}
func newBufConn(conn net.Conn) *bufConn {
return &bufConn{
Conn: conn,
buf: bufio.NewReader(conn),
}
}
func (bc *bufConn) Read(b []byte) (n int, err error) {
return bc.buf.Read(b)
}
// KafkaVersion instances represent versions of the upstream Kafka broker.
type KafkaVersion struct {
// it's a struct rather than just typing the array directly to make it opaque and stop people
// generating their own arbitrary versions
version [4]uint
}
func newKafkaVersion(major, minor, veryMinor, patch uint) KafkaVersion {
return KafkaVersion{
version: [4]uint{major, minor, veryMinor, patch},
}
}
// IsAtLeast return true if and only if the version it is called on is
// greater than or equal to the version passed in:
//
// V1.IsAtLeast(V2) // false
// V2.IsAtLeast(V1) // true
func (v KafkaVersion) IsAtLeast(other KafkaVersion) bool {
for i := range v.version {
if v.version[i] > other.version[i] {
return true
} else if v.version[i] < other.version[i] {
return false
}
}
return true
}
// Effective constants defining the supported kafka versions.
var (
V0_8_2_0 = newKafkaVersion(0, 8, 2, 0)
V0_8_2_1 = newKafkaVersion(0, 8, 2, 1)
V0_8_2_2 = newKafkaVersion(0, 8, 2, 2)
V0_9_0_0 = newKafkaVersion(0, 9, 0, 0)
V0_9_0_1 = newKafkaVersion(0, 9, 0, 1)
V0_10_0_0 = newKafkaVersion(0, 10, 0, 0)
V0_10_0_1 = newKafkaVersion(0, 10, 0, 1)
V0_10_1_0 = newKafkaVersion(0, 10, 1, 0)
V0_10_1_1 = newKafkaVersion(0, 10, 1, 1)
V0_10_2_0 = newKafkaVersion(0, 10, 2, 0)
V0_10_2_1 = newKafkaVersion(0, 10, 2, 1)
V0_10_2_2 = newKafkaVersion(0, 10, 2, 2)
V0_11_0_0 = newKafkaVersion(0, 11, 0, 0)
V0_11_0_1 = newKafkaVersion(0, 11, 0, 1)
V0_11_0_2 = newKafkaVersion(0, 11, 0, 2)
V1_0_0_0 = newKafkaVersion(1, 0, 0, 0)
V1_0_1_0 = newKafkaVersion(1, 0, 1, 0)
V1_0_2_0 = newKafkaVersion(1, 0, 2, 0)
V1_1_0_0 = newKafkaVersion(1, 1, 0, 0)
V1_1_1_0 = newKafkaVersion(1, 1, 1, 0)
V2_0_0_0 = newKafkaVersion(2, 0, 0, 0)
V2_0_1_0 = newKafkaVersion(2, 0, 1, 0)
V2_1_0_0 = newKafkaVersion(2, 1, 0, 0)
V2_1_1_0 = newKafkaVersion(2, 1, 1, 0)
V2_2_0_0 = newKafkaVersion(2, 2, 0, 0)
V2_2_1_0 = newKafkaVersion(2, 2, 1, 0)
V2_2_2_0 = newKafkaVersion(2, 2, 2, 0)
V2_3_0_0 = newKafkaVersion(2, 3, 0, 0)
V2_3_1_0 = newKafkaVersion(2, 3, 1, 0)
V2_4_0_0 = newKafkaVersion(2, 4, 0, 0)
V2_4_1_0 = newKafkaVersion(2, 4, 1, 0)
V2_5_0_0 = newKafkaVersion(2, 5, 0, 0)
V2_5_1_0 = newKafkaVersion(2, 5, 1, 0)
V2_6_0_0 = newKafkaVersion(2, 6, 0, 0)
V2_6_1_0 = newKafkaVersion(2, 6, 1, 0)
V2_6_2_0 = newKafkaVersion(2, 6, 2, 0)
V2_6_3_0 = newKafkaVersion(2, 6, 3, 0)
V2_7_0_0 = newKafkaVersion(2, 7, 0, 0)
V2_7_1_0 = newKafkaVersion(2, 7, 1, 0)
V2_7_2_0 = newKafkaVersion(2, 7, 2, 0)
V2_8_0_0 = newKafkaVersion(2, 8, 0, 0)
V2_8_1_0 = newKafkaVersion(2, 8, 1, 0)
V2_8_2_0 = newKafkaVersion(2, 8, 2, 0)
V3_0_0_0 = newKafkaVersion(3, 0, 0, 0)
V3_0_1_0 = newKafkaVersion(3, 0, 1, 0)
V3_0_2_0 = newKafkaVersion(3, 0, 2, 0)
V3_1_0_0 = newKafkaVersion(3, 1, 0, 0)
V3_1_1_0 = newKafkaVersion(3, 1, 1, 0)
V3_1_2_0 = newKafkaVersion(3, 1, 2, 0)
V3_2_0_0 = newKafkaVersion(3, 2, 0, 0)
V3_2_1_0 = newKafkaVersion(3, 2, 1, 0)
V3_2_2_0 = newKafkaVersion(3, 2, 2, 0)
V3_2_3_0 = newKafkaVersion(3, 2, 3, 0)
V3_3_0_0 = newKafkaVersion(3, 3, 0, 0)
V3_3_1_0 = newKafkaVersion(3, 3, 1, 0)
V3_3_2_0 = newKafkaVersion(3, 3, 2, 0)
V3_4_0_0 = newKafkaVersion(3, 4, 0, 0)
V3_4_1_0 = newKafkaVersion(3, 4, 1, 0)
V3_5_0_0 = newKafkaVersion(3, 5, 0, 0)
V3_5_1_0 = newKafkaVersion(3, 5, 1, 0)
V3_6_0_0 = newKafkaVersion(3, 6, 0, 0)
SupportedVersions = []KafkaVersion{
V0_8_2_0,
V0_8_2_1,
V0_8_2_2,
V0_9_0_0,
V0_9_0_1,
V0_10_0_0,
V0_10_0_1,
V0_10_1_0,
V0_10_1_1,
V0_10_2_0,
V0_10_2_1,
V0_10_2_2,
V0_11_0_0,
V0_11_0_1,
V0_11_0_2,
V1_0_0_0,
V1_0_1_0,
V1_0_2_0,
V1_1_0_0,
V1_1_1_0,
V2_0_0_0,
V2_0_1_0,
V2_1_0_0,
V2_1_1_0,
V2_2_0_0,
V2_2_1_0,
V2_2_2_0,
V2_3_0_0,
V2_3_1_0,
V2_4_0_0,
V2_4_1_0,
V2_5_0_0,
V2_5_1_0,
V2_6_0_0,
V2_6_1_0,
V2_6_2_0,
V2_7_0_0,
V2_7_1_0,
V2_8_0_0,
V2_8_1_0,
V2_8_2_0,
V3_0_0_0,
V3_0_1_0,
V3_0_2_0,
V3_1_0_0,
V3_1_1_0,
V3_1_2_0,
V3_2_0_0,
V3_2_1_0,
V3_2_2_0,
V3_2_3_0,
V3_3_0_0,
V3_3_1_0,
V3_3_2_0,
V3_4_0_0,
V3_4_1_0,
V3_5_0_0,
V3_5_1_0,
V3_6_0_0,
}
MinVersion = V0_8_2_0
MaxVersion = V3_6_0_0
DefaultVersion = V2_1_0_0
// reduced set of protocol versions to matrix test
fvtRangeVersions = []KafkaVersion{
V0_8_2_2,
V0_10_2_2,
V1_0_2_0,
V1_1_1_0,
V2_0_1_0,
V2_2_2_0,
V2_4_1_0,
V2_6_2_0,
V2_8_2_0,
V3_1_2_0,
V3_3_2_0,
V3_6_0_0,
}
)
var (
// This regex validates that a string complies with the pre kafka 1.0.0 format for version strings, for example 0.11.0.3
validPreKafka1Version = regexp.MustCompile(`^0\.\d+\.\d+\.\d+$`)
// This regex validates that a string complies with the post Kafka 1.0.0 format, for example 1.0.0
validPostKafka1Version = regexp.MustCompile(`^\d+\.\d+\.\d+$`)
)
// ParseKafkaVersion parses and returns kafka version or error from a string
func ParseKafkaVersion(s string) (KafkaVersion, error) {
if len(s) < 5 {
return DefaultVersion, fmt.Errorf("invalid version `%s`", s)
}
var major, minor, veryMinor, patch uint
var err error
if s[0] == '0' {
err = scanKafkaVersion(s, validPreKafka1Version, "0.%d.%d.%d", [3]*uint{&minor, &veryMinor, &patch})
} else {
err = scanKafkaVersion(s, validPostKafka1Version, "%d.%d.%d", [3]*uint{&major, &minor, &veryMinor})
}
if err != nil {
return DefaultVersion, err
}
return newKafkaVersion(major, minor, veryMinor, patch), nil
}
func scanKafkaVersion(s string, pattern *regexp.Regexp, format string, v [3]*uint) error {
if !pattern.MatchString(s) {
return fmt.Errorf("invalid version `%s`", s)
}
_, err := fmt.Sscanf(s, format, v[0], v[1], v[2])
return err
}
func (v KafkaVersion) String() string {
if v.version[0] == 0 {
return fmt.Sprintf("0.%d.%d.%d", v.version[1], v.version[2], v.version[3])
}
return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2])
}
golang-github-ibm-sarama-1.43.2/utils_test.go 0000664 0000000 0000000 00000003176 14612567413 0021105 0 ustar 00root root 0000000 0000000 package sarama
import "testing"
func TestVersionCompare(t *testing.T) {
if V0_8_2_0.IsAtLeast(V0_8_2_1) {
t.Error("0.8.2.0 >= 0.8.2.1")
}
if !V0_8_2_1.IsAtLeast(V0_8_2_0) {
t.Error("! 0.8.2.1 >= 0.8.2.0")
}
if !V0_8_2_0.IsAtLeast(V0_8_2_0) {
t.Error("! 0.8.2.0 >= 0.8.2.0")
}
if !V0_9_0_0.IsAtLeast(V0_8_2_1) {
t.Error("! 0.9.0.0 >= 0.8.2.1")
}
if V0_8_2_1.IsAtLeast(V0_10_0_0) {
t.Error("0.8.2.1 >= 0.10.0.0")
}
if !V1_0_0_0.IsAtLeast(V0_9_0_0) {
t.Error("! 1.0.0.0 >= 0.9.0.0")
}
if V0_9_0_0.IsAtLeast(V1_0_0_0) {
t.Error("0.9.0.0 >= 1.0.0.0")
}
}
func TestVersionParsing(t *testing.T) {
validVersions := []string{
"0.8.2.0",
"0.8.2.1",
"0.8.2.2",
"0.9.0.0",
"0.9.0.1",
"0.10.0.0",
"0.10.0.1",
"0.10.1.0",
"0.10.1.1",
"0.10.2.0",
"0.10.2.1",
"0.10.2.2",
"0.11.0.0",
"0.11.0.1",
"0.11.0.2",
"1.0.0",
"1.0.1",
"1.0.2",
"1.1.0",
"1.1.1",
"2.0.0",
"2.0.1",
"2.1.0",
"2.1.1",
"2.2.0",
"2.2.1",
"2.2.2",
"2.3.0",
"2.3.1",
"2.4.0",
"2.4.1",
"2.5.0",
"2.5.1",
"2.6.0",
"2.6.1",
"2.6.2",
"2.6.3",
"2.7.0",
"2.7.1",
"2.7.2",
"2.8.0",
"2.8.1",
"3.0.0",
"3.0.1",
"3.1.0",
"3.1.1",
"3.2.0",
}
for _, s := range validVersions {
v, err := ParseKafkaVersion(s)
if err != nil {
t.Errorf("could not parse valid version %s: %s", s, err)
}
if v.String() != s {
t.Errorf("version %s != %s", v.String(), s)
}
}
invalidVersions := []string{"0.8.2-4", "0.8.20", "1.19.0.0", "1.0.x"}
for _, s := range invalidVersions {
if _, err := ParseKafkaVersion(s); err == nil {
t.Errorf("invalid version %s parsed without error", s)
}
}
}
golang-github-ibm-sarama-1.43.2/version.go 0000664 0000000 0000000 00000000717 14612567413 0020371 0 ustar 00root root 0000000 0000000 package sarama
import (
"runtime/debug"
"sync"
)
var (
v string
vOnce sync.Once
)
func version() string {
vOnce.Do(func() {
bi, ok := debug.ReadBuildInfo()
if ok {
v = bi.Main.Version
}
if v == "" || v == "(devel)" {
// if we can't read a go module version then they're using a git
// clone or vendored module so all we can do is report "dev" for
// the version to make a valid ApiVersions request
v = "dev"
}
})
return v
}
golang-github-ibm-sarama-1.43.2/zstd.go 0000664 0000000 0000000 00000003655 14612567413 0017674 0 ustar 00root root 0000000 0000000 package sarama
import (
"sync"
"github.com/klauspost/compress/zstd"
)
// zstdMaxBufferedEncoders maximum number of not-in-use zstd encoders
// If the pool of encoders is exhausted then new encoders will be created on the fly
const zstdMaxBufferedEncoders = 1
type ZstdEncoderParams struct {
Level int
}
type ZstdDecoderParams struct {
}
var zstdDecMap sync.Map
var zstdAvailableEncoders sync.Map
func getZstdEncoderChannel(params ZstdEncoderParams) chan *zstd.Encoder {
if c, ok := zstdAvailableEncoders.Load(params); ok {
return c.(chan *zstd.Encoder)
}
c, _ := zstdAvailableEncoders.LoadOrStore(params, make(chan *zstd.Encoder, zstdMaxBufferedEncoders))
return c.(chan *zstd.Encoder)
}
func getZstdEncoder(params ZstdEncoderParams) *zstd.Encoder {
select {
case enc := <-getZstdEncoderChannel(params):
return enc
default:
encoderLevel := zstd.SpeedDefault
if params.Level != CompressionLevelDefault {
encoderLevel = zstd.EncoderLevelFromZstd(params.Level)
}
zstdEnc, _ := zstd.NewWriter(nil, zstd.WithZeroFrames(true),
zstd.WithEncoderLevel(encoderLevel),
zstd.WithEncoderConcurrency(1))
return zstdEnc
}
}
func releaseEncoder(params ZstdEncoderParams, enc *zstd.Encoder) {
select {
case getZstdEncoderChannel(params) <- enc:
default:
}
}
func getDecoder(params ZstdDecoderParams) *zstd.Decoder {
if ret, ok := zstdDecMap.Load(params); ok {
return ret.(*zstd.Decoder)
}
// It's possible to race and create multiple new readers.
// Only one will survive GC after use.
zstdDec, _ := zstd.NewReader(nil, zstd.WithDecoderConcurrency(0))
zstdDecMap.Store(params, zstdDec)
return zstdDec
}
func zstdDecompress(params ZstdDecoderParams, dst, src []byte) ([]byte, error) {
return getDecoder(params).DecodeAll(src, dst)
}
func zstdCompress(params ZstdEncoderParams, dst, src []byte) ([]byte, error) {
enc := getZstdEncoder(params)
out := enc.EncodeAll(src, dst)
releaseEncoder(params, enc)
return out, nil
}
golang-github-ibm-sarama-1.43.2/zstd_test.go 0000664 0000000 0000000 00000001136 14612567413 0020723 0 ustar 00root root 0000000 0000000 package sarama
import (
"runtime"
"testing"
)
func BenchmarkZstdMemoryConsumption(b *testing.B) {
params := ZstdEncoderParams{Level: 9}
buf := make([]byte, 1024*1024)
for i := 0; i < len(buf); i++ {
buf[i] = byte((i / 256) + (i * 257))
}
cpus := 96
gomaxprocsBackup := runtime.GOMAXPROCS(cpus)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
for j := 0; j < 2*cpus; j++ {
_, _ = zstdCompress(params, nil, buf)
}
// drain the buffered encoder
getZstdEncoder(params)
// previously this would be achieved with
// zstdEncMap.Delete(params)
}
runtime.GOMAXPROCS(gomaxprocsBackup)
}