pax_global_header00006660000000000000000000000064134335734600014522gustar00rootroot0000000000000052 comment=e792853782b2ba1dd303c414460448a462360ece burrow-1.2.1/000077500000000000000000000000001343357346000130435ustar00rootroot00000000000000burrow-1.2.1/.gitignore000066400000000000000000000001431343357346000150310ustar00rootroot00000000000000burrow-src .*.swp !config dist log .idea *.cov Burrow Burrow.exe Burrow.iml tmp vendor burrow-1.2.1/.goreleaser.yml000066400000000000000000000013751343357346000160020ustar00rootroot00000000000000# .goreleaser.yml # Build customization builds: - main: main.go binary: burrow goos: - windows - darwin - linux goarch: - amd64 # Archive customization archive: format: tar.gz files: - LICENSE - NOTICE - README.md - CHANGELOG.md - config/burrow.toml - config/default-email.tmpl - config/default-http-delete.tmpl - config/default-http-post.tmpl - config/default-slack-delete.tmpl - config/default-slack-post.tmpl dockers: - goos: linux goarch: amd64 goarm: '' binaries: - burrow dockerfile: Dockerfile.gorelease image_templates: - 'toddpalino/burrow:latest' - 'toddpalino/burrow:{{ .Tag }}' extra_files: - docker-config/burrow.toml burrow-1.2.1/.travis.yml000066400000000000000000000051221343357346000151540ustar00rootroot00000000000000language: go go: - 1.10.8 - 1.11.5 - master env: - DEP_VERSION="0.5.0" sudo: required services: - docker before_install: # Download the binary to bin folder in $GOPATH - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep # Make the binary executable - chmod +x $GOPATH/bin/dep install: - dep ensure matrix: # It's ok if our code fails on unstable development versions of Go. allow_failures: - go: master # Don't wait for tip tests to finish. Mark the test run green if the # tests pass on the stable versions of Go. fast_finish: true # Don't email me the results of the test runs. notifications: email: false webhooks: urls: - https://webhooks.gitter.im/e/ec30ac1bb280c976f8be on_success: change # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always on_start: never # options: [always|never|change] default: always # Anything in before_script that returns a nonzero exit code will # flunk the build and immediately stop. It's sorta like having # set -e enabled in bash. before_script: - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/) # All the .go files, excluding vendor/ - go get golang.org/x/lint/golint # Linter - go get honnef.co/go/tools/cmd/staticcheck # Badass static analyzer/linter - go get github.com/fzipp/gocyclo - go get github.com/mattn/goveralls # script always run to completion (set +e). All of these code checks are must haves # in a modern Go project. script: - test -z $(gofmt -s -l $GO_FILES) # Fail if a .go file hasn't been formatted with gofmt - go test --timeout 5s -v -race ./... # Run all the tests with the race detector enabled - ./testAndCover.sh # Run the tests again to get coverage info - go vet -composites=false ./... # go vet is the official Go static analyzer - staticcheck ./... # "go vet on steroids" + linter - gocyclo -over 15 $GO_FILES # forbid code with huge functions - golint -set_exit_status $(go list ./...) # one last linter # If the build succeeds, send coverage to coveralls # goreleaser will run if the latest version tag matches the current commit after_success: - $GOPATH/bin/goveralls -coverprofile=profile.cov -service=travis-ci - if [[ "$TRAVIS_SECURE_ENV_VARS" == true && "$TRAVIS_GO_VERSION" == 1.11* ]]; then docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; curl -sL https://git.io/goreleaser | bash; fi burrow-1.2.1/CHANGELOG.md000066400000000000000000000106171343357346000146610ustar00rootroot00000000000000## 1.2.1 (2019-02-21) **Release Highlights** * Fix binary release process. * Report `ClientID` for consumers. * Fix division by zero error in evaluator. ## 1.2.0 (2019-01-18) **Release Highlights** * Add support for Kafka up to version 2.1.0. * Update sarama to version 1.20.1 with support for zstd compression. * Support linux/arm64. * Add blacklist for memory store. **Changes** * [[`d244fce922`](https://github.com/nodejs/node/commit/d244fce922)] - Bump sarama to 1.20.1 (Vlad Gorodetsky) * [[`793430d249`](https://github.com/nodejs/node/commit/793430d249)] - Golang 1.9.x is no longer supported (Vlad Gorodetsky) * [[`735fcb7c82`](https://github.com/nodejs/node/commit/735fcb7c82)] - Replace deprecated megacheck with staticcheck (Vlad Gorodetsky) * [[`3d49b2588b`](https://github.com/nodejs/node/commit/3d49b2588b)] - Link the README to the Compose file in the project (Jordan Moore) * [[`3a59b36d94`](https://github.com/nodejs/node/commit/3a59b36d94)] - Tests fixed (Mikhail Chugunkov) * [[`6684c5e4db`](https://github.com/nodejs/node/commit/6684c5e4db)] - Added unit test for v3 value decoding (Mikhail Chugunkov) * [[`10d4dc39eb`](https://github.com/nodejs/node/commit/10d4dc39eb)] - Added v3 messages protocol support (Mikhail Chugunkov) * [[`d6b075b781`](https://github.com/nodejs/node/commit/d6b075b781)] - Replace deprecated MAINTAINER directive with a label (Vlad Gorodetsky) * [[`52606499a6`](https://github.com/nodejs/node/commit/52606499a6)] - Refactor parseKafkaVersion to reduce method complexity (gocyclo) (Vlad Gorodetsky) * [[`b0440f9dea`](https://github.com/nodejs/node/commit/b0440f9dea)] - Add gcc to build zstd (Vlad Gorodetsky) * [[`6898a8de26`](https://github.com/nodejs/node/commit/6898a8de26)] - Add libc-dev to build zstd (Vlad Gorodetsky) * [[`b81089aada`](https://github.com/nodejs/node/commit/b81089aada)] - Add support for Kafka 2.1.0 (Vlad Gorodetsky) * [[`cb004f9405`](https://github.com/nodejs/node/commit/cb004f9405)] - Build with Go 1.11 (Vlad Gorodetsky) * [[`679a95fb38`](https://github.com/nodejs/node/commit/679a95fb38)] - Fix golint import path (golint fixer) * [[`f88bb7d3a8`](https://github.com/nodejs/node/commit/f88bb7d3a8)] - Update docker-compose Readme section with working url. (Daniel Wojda) * [[`3f888cdb2d`](https://github.com/nodejs/node/commit/3f888cdb2d)] - Upgrade sarama to support Kafka 2.0.0 (#440) (daniel) * [[`1150f6fef9`](https://github.com/nodejs/node/commit/1150f6fef9)] - Support linux/arm64 using Dup3() instead of Dup2() (Mpampis Kostas) * [[`1b65b4b2f2`](https://github.com/nodejs/node/commit/1b65b4b2f2)] - Add support for Kafka 1.1.0 (#403) (Vlad Gorodetsky) * [[`74b309fc8d`](https://github.com/nodejs/node/commit/74b309fc8d)] - code coverage for newly added lines (Clemens Valiente) * [[`279c75375c`](https://github.com/nodejs/node/commit/279c75375c)] - accidentally reverted this (Clemens Valiente) * [[`192878c69c`](https://github.com/nodejs/node/commit/192878c69c)] - gofmt (Clemens Valiente) * [[`33bc8defcd`](https://github.com/nodejs/node/commit/33bc8defcd)] - make first regex test case a proper match everything (Clemens Valiente) * [[`279b256b27`](https://github.com/nodejs/node/commit/279b256b27)] - only set whitelist / blacklist if it's not empty string (Clemens Valiente) * [[`b48d30d18c`](https://github.com/nodejs/node/commit/b48d30d18c)] - naming (Clemens Valiente) * [[`7d6c6ccb03`](https://github.com/nodejs/node/commit/7d6c6ccb03)] - variable naming (Clemens Valiente) * [[`4e051e973f`](https://github.com/nodejs/node/commit/4e051e973f)] - add tests (Clemens Valiente) * [[`545bec66d0`](https://github.com/nodejs/node/commit/545bec66d0)] - add blacklist for memory store (Clemens Valiente) * [[`07af26d2f1`](https://github.com/nodejs/node/commit/07af26d2f1)] - Updated burrow endpoint in README : #401 (Ratish Ravindran) * [[`fecab1ea88`](https://github.com/nodejs/node/commit/fecab1ea88)] - pass custom headers to http notifications. (#357) (vixns) ## 1.0.0 (TBD) Features: - Code overhaul - more modular and now with tests - Actual documentation (godoc) - Support for topic deletion in Kafka clusters - Removed Slack notifier in favor of just using the HTTP notifier Bugfixes: - Too many to count ## 0.1.1 (2016-05-01) Features: - ZK Offset checking - Storm offset checking Bugfixes: - Fixed an issue with not closing HTTP requests and responses properly - Change internal hostname structures for properly support IPv6 ## 0.1.0 (2015-10-07) Initial version release burrow-1.2.1/Dockerfile000066400000000000000000000012501343357346000150330ustar00rootroot00000000000000FROM golang:1.11-alpine as builder ENV DEP_VERSION="0.5.0" RUN apk add --no-cache git curl gcc libc-dev && \ curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep && \ chmod +x $GOPATH/bin/dep && \ mkdir -p $GOPATH/src/github.com/linkedin/Burrow ADD . $GOPATH/src/github.com/linkedin/Burrow/ RUN cd $GOPATH/src/github.com/linkedin/Burrow && \ dep ensure && \ go build -o /tmp/burrow . FROM iron/go LABEL maintainer="LinkedIn Burrow https://github.com/linkedin/Burrow" WORKDIR /app COPY --from=builder /tmp/burrow /app/ ADD /docker-config/burrow.toml /etc/burrow/ CMD ["/app/burrow", "--config-dir", "/etc/burrow"] burrow-1.2.1/Dockerfile.gorelease000066400000000000000000000002751343357346000170060ustar00rootroot00000000000000FROM iron/go MAINTAINER LinkedIn Burrow "https://github.com/linkedin/Burrow" WORKDIR /app ADD burrow /app/ ADD burrow.toml /etc/burrow/ CMD ["/app/burrow", "--config-dir", "/etc/burrow"] burrow-1.2.1/Gopkg.lock000066400000000000000000000227231343357346000147720ustar00rootroot00000000000000# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. [[projects]] digest = "1:29b1e6604e762715716cae79145c8732a964b8fe564cc330de1495090fbc777a" name = "github.com/DataDog/zstd" packages = ["."] pruneopts = "" revision = "c7161f8c63c045cbc7ca051dcc969dd0e4054de2" version = "v1.3.5" [[projects]] digest = "1:3aa363b7d4e2990b3d863ba0d7cdc111c9513d0d33439bbda1eba266c094a848" name = "github.com/OneOfOne/xxhash" packages = ["."] pruneopts = "" revision = "6def279d2ce6c81a79dd1c1be580f03bb216fb8a" version = "v1.2.2" [[projects]] digest = "1:072c4df72b72758253d774fe5602c1a9ab86056e55ec806def5aa139e5ac7a4d" name = "github.com/Shopify/sarama" packages = ["."] pruneopts = "" revision = "03a43f93cd29dc549e6d9b11892795c206f9c38c" version = "v1.20.1" [[projects]] digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77" name = "github.com/davecgh/go-spew" packages = ["spew"] pruneopts = "" revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" version = "v1.1.1" [[projects]] digest = "1:6d6672f85a84411509885eaa32f597577873de00e30729b9bb0eb1e1faa49c12" name = "github.com/eapache/go-resiliency" packages = ["breaker"] pruneopts = "" revision = "ea41b0fad31007accc7f806884dcdf3da98b79ce" version = "v1.1.0" [[projects]] branch = "master" digest = "1:6643c01e619a68f80ac12ad81223275df653528c6d7e3788291c1fd6f1d622f6" name = "github.com/eapache/go-xerial-snappy" packages = ["."] pruneopts = "" revision = "776d5712da21bc4762676d614db1d8a64f4238b0" [[projects]] digest = "1:d8d46d21073d0f65daf1740ebf4629c65e04bf92e14ce93c2201e8624843c3d3" name = "github.com/eapache/queue" packages = ["."] pruneopts = "" revision = "44cc805cf13205b55f69e14bcb69867d1ae92f98" version = "v1.1.0" [[projects]] digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd" name = "github.com/fsnotify/fsnotify" packages = ["."] pruneopts = "" revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" [[projects]] branch = "master" digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf" name = "github.com/golang/snappy" packages = ["."] pruneopts = "" revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" [[projects]] digest = "1:a25a2c5ae694b01713fb6cd03c3b1ac1ccc1902b9f0a922680a88ec254f968e1" name = "github.com/google/uuid" packages = ["."] pruneopts = "" revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8" version = "v1.1.0" [[projects]] digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec" name = "github.com/hashicorp/hcl" packages = [ ".", "hcl/ast", "hcl/parser", "hcl/printer", "hcl/scanner", "hcl/strconv", "hcl/token", "json/parser", "json/scanner", "json/token", ] pruneopts = "" revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" version = "v1.0.0" [[projects]] branch = "master" digest = "1:1c2fc412375033d9b9f3f7fc5e9554d94c9a82a042c6e03d03ccc207e9f14928" name = "github.com/julienschmidt/httprouter" packages = ["."] pruneopts = "" revision = "26a05976f9bf5c3aa992cc20e8588c359418ee58" [[projects]] digest = "1:3e6cf98fa86629a89185762c6a52c996a88c4c9c2bf04e558056d55331a15a7e" name = "github.com/karrick/goswarm" packages = ["."] pruneopts = "" revision = "0e5d241bdd481ab21611a8fadcddcdb1a810536e" version = "v1.9.1" [[projects]] digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5" name = "github.com/magiconair/properties" packages = ["."] pruneopts = "" revision = "c2353362d570a7bfa228149c62842019201cfb71" version = "v1.8.0" [[projects]] digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60" name = "github.com/mitchellh/mapstructure" packages = ["."] pruneopts = "" revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" version = "v1.1.2" [[projects]] digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e" name = "github.com/pborman/uuid" packages = ["."] pruneopts = "" revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1" version = "v1.2" [[projects]] digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457" name = "github.com/pelletier/go-toml" packages = ["."] pruneopts = "" revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" version = "v1.2.0" [[projects]] digest = "1:b1df71d0b2287062b90c6b4c8d3c934440aa0d2eb201d03f22be0f045860b4aa" name = "github.com/pierrec/lz4" packages = [ ".", "internal/xxh32", ] pruneopts = "" revision = "635575b42742856941dbc767b44905bb9ba083f6" version = "v2.0.7" [[projects]] digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" name = "github.com/pmezard/go-difflib" packages = ["difflib"] pruneopts = "" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] branch = "master" digest = "1:bc5884d890d71ae56382665a93d792af3602dae40fa98778170e4795598a7264" name = "github.com/rcrowley/go-metrics" packages = ["."] pruneopts = "" revision = "3113b8401b8a98917cde58f8bbd42a1b1c03b1fd" [[projects]] branch = "master" digest = "1:7fc2f428767a2521abc63f1a663d981f61610524275d6c0ea645defadd4e916f" name = "github.com/samuel/go-zookeeper" packages = ["zk"] pruneopts = "" revision = "c4fab1ac1bec58281ad0667dc3f0907a9476ac47" [[projects]] digest = "1:32c5802989a96ee74cc4e8372efce3cab9cf6355132ad8e80cc32c18757ccd47" name = "github.com/spf13/afero" packages = [ ".", "mem", ] pruneopts = "" revision = "a5d6946387efe7d64d09dcba68cdd523dc1273a3" version = "v1.2.0" [[projects]] digest = "1:ae3493c780092be9d576a1f746ab967293ec165e8473425631f06658b6212afc" name = "github.com/spf13/cast" packages = ["."] pruneopts = "" revision = "8c9545af88b134710ab1cd196795e7f2388358d7" version = "v1.3.0" [[projects]] digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0" name = "github.com/spf13/jwalterweatherman" packages = ["."] pruneopts = "" revision = "4a4406e478ca629068e7768fc33f3f044173c0a6" version = "v1.0.0" [[projects]] digest = "1:cbaf13cdbfef0e4734ed8a7504f57fe893d471d62a35b982bf6fb3f036449a66" name = "github.com/spf13/pflag" packages = ["."] pruneopts = "" revision = "298182f68c66c05229eb03ac171abe6e309ee79a" version = "v1.0.3" [[projects]] digest = "1:b98ee2c3f1469ab3b494859d3a9c9e595ec2c63660dea130a5154cfcd427b608" name = "github.com/spf13/viper" packages = ["."] pruneopts = "" revision = "6d33b5a963d922d182c91e8a1c88d81fd150cfd4" version = "v1.3.1" [[projects]] digest = "1:711eebe744c0151a9d09af2315f0bb729b2ec7637ef4c410fa90a18ef74b65b6" name = "github.com/stretchr/objx" packages = ["."] pruneopts = "" revision = "477a77ecc69700c7cdeb1fa9e129548e1c1c393c" version = "v0.1.1" [[projects]] branch = "master" digest = "1:a7cd8604f0b92d5a43f723b6f13b6650cf2f6be5e40235a7c4f3cc5951f657bd" name = "github.com/stretchr/testify" packages = [ "assert", "mock", ] pruneopts = "" revision = "363ebb24d041ccea8068222281c2e963e997b9dc" [[projects]] digest = "1:74f86c458e82e1c4efbab95233e0cf51b7cc02dc03193be9f62cd81224e10401" name = "go.uber.org/atomic" packages = ["."] pruneopts = "" revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289" version = "v1.3.2" [[projects]] digest = "1:22c7effcb4da0eacb2bb1940ee173fac010e9ef3c691f5de4b524d538bd980f5" name = "go.uber.org/multierr" packages = ["."] pruneopts = "" revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a" version = "v1.1.0" [[projects]] digest = "1:246f378f80fba6fcf0f191c486b6613265abd2bc0f2fa55a36b928c67352021e" name = "go.uber.org/zap" packages = [ ".", "buffer", "internal/bufferpool", "internal/color", "internal/exit", "zapcore", ] pruneopts = "" revision = "ff33455a0e382e8a81d14dd7c922020b6b5e7982" version = "v1.9.1" [[projects]] branch = "master" digest = "1:057806b6e218f4ceac80d27ccd8dd80b2c06b0050289aa46e0edfd82cc4faa2c" name = "golang.org/x/sys" packages = ["unix"] pruneopts = "" revision = "11f53e03133963fb11ae0588e08b5e0b85be8be5" [[projects]] digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4" name = "golang.org/x/text" packages = [ "internal/gen", "internal/triegen", "internal/ucd", "transform", "unicode/cldr", "unicode/norm", ] pruneopts = "" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] digest = "1:11c58e19ff7ce22740423bb933f1ddca3bf575def40d5ac3437ec12871b1648b" name = "gopkg.in/natefinch/lumberjack.v2" packages = ["."] pruneopts = "" revision = "a96e63847dc3c67d17befa69c303767e2f84e54f" version = "v2.1" [[projects]] digest = "1:cedccf16b71e86db87a24f8d4c70b0a855872eb967cb906a66b95de56aefbd0d" name = "gopkg.in/yaml.v2" packages = ["."] pruneopts = "" revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" version = "v2.2.2" [solve-meta] analyzer-name = "dep" analyzer-version = 1 input-imports = [ "github.com/OneOfOne/xxhash", "github.com/Shopify/sarama", "github.com/julienschmidt/httprouter", "github.com/karrick/goswarm", "github.com/pborman/uuid", "github.com/samuel/go-zookeeper/zk", "github.com/spf13/viper", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/mock", "go.uber.org/zap", "go.uber.org/zap/zapcore", "gopkg.in/natefinch/lumberjack.v2", ] solver-name = "gps-cdcl" solver-version = 1 burrow-1.2.1/Gopkg.toml000066400000000000000000000013431343357346000150100ustar00rootroot00000000000000[[constraint]] name = "github.com/Shopify/sarama" version = "1.20.1" [[constraint]] name = "go.uber.org/zap" version = "1.7.1" [[constraint]] name = "gopkg.in/natefinch/lumberjack.v2" version = "2.1" [[constraint]] name = "github.com/pborman/uuid" version = "1.1.0" [[constraint]] branch = "master" name = "github.com/samuel/go-zookeeper" [[constraint]] name = "github.com/julienschmidt/httprouter" branch = "master" [[constraint]] name = "github.com/stretchr/testify" branch = "master" [[constraint]] name = "github.com/karrick/goswarm" version = "1.4.7" [[constraint]] name = "github.com/OneOfOne/xxhash" version = "1.2.1" [[constraint]] name = "github.com/spf13/viper" version = "1.0.0" burrow-1.2.1/LICENSE000066400000000000000000000261361343357346000140600ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. burrow-1.2.1/NOTICE000066400000000000000000000030711343357346000137500ustar00rootroot00000000000000Copyright 2015 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. This product includes/uses Go (https://golang.org/) Copyright (C) 2012 The Go Authors License: BSD This product includes/uses gcfg (https://code.google.com/p/gcfg) Copyright (c) 2012 Péter Surányi License: BSD This product includes/uses go-uuid (https://code.google.com/p/go-uuid) Copyright (c) 2009,2014 Google Inc. License: BSD This product includes/uses snappy-go (https://code.google.com/p/snappy-go) Copyright (c) 2011 The Snappy-Go Authors License: BSD This product includes/uses cihub/seelog (https://github.com/cihub/seelog) Copyright (c) 2012, Cloud Instruments Co., Ltd. License: BSD This product includes/uses eapache/go-resiliency (https://github.com/eapache/go-resiliency/) Copyright (c) 2014 Evan Huus License: MIT This product includes/uses eapache/queue (https://github.com/eapache/queue/) Copyright (c) 2014 Evan Huus License: MIT This product includes/uses go-zookeeper (https://github.com/samuel/go-zookeeper/) Copyright (c) 2013, Samuel Stauffer License: BSD This product includes/uses sarama (https://github.com/Shopify/sarama/) Copyright (c) 2013 Evan Huus License: MIT burrow-1.2.1/README.md000066400000000000000000000072271343357346000143320ustar00rootroot00000000000000[![Join the chat at https://gitter.im/linkedin-Burrow/Lobby](https://badges.gitter.im/linkedin-Burrow/Lobby.svg)](https://gitter.im/linkedin-Burrow/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/linkedin/Burrow.svg)](https://travis-ci.org/linkedin/Burrow) [![go report card](https://goreportcard.com/badge/github.com/linkedin/Burrow)](https://goreportcard.com/report/github.com/linkedin/Burrow) [![Coverage Status](https://coveralls.io/repos/github/linkedin/Burrow/badge.svg?branch=master)](https://coveralls.io/github/linkedin/Burrow?branch=master) [![GoDoc](https://godoc.org/github.com/linkedin/Burrow?status.svg)](https://godoc.org/github.com/linkedin/Burrow) # Burrow - Kafka Consumer Lag Checking Burrow is a monitoring companion for [Apache Kafka](http://kafka.apache.org) that provides consumer lag checking as a service without the need for specifying thresholds. It monitors committed offsets for all consumers and calculates the status of those consumers on demand. An HTTP endpoint is provided to request status on demand, as well as provide other Kafka cluster information. There are also configurable notifiers that can send status out via email or HTTP calls to another service. ## Features * NO THRESHOLDS! Groups are evaluated over a sliding window. * Multiple Kafka Cluster support * Automatically monitors all consumers using Kafka-committed offsets * Configurable support for Zookeeper-committed offsets * Configurable support for Storm-committed offsets * HTTP endpoint for consumer group status, as well as broker and consumer information * Configurable emailer for sending alerts for specific groups * Configurable HTTP client for sending alerts to another system for all groups ## Getting Started ### Prerequisites Burrow is written in Go, so before you get started, you should [install and set up Go](https://golang.org/doc/install). If you have not yet installed the [Go Dependency Management Tool](https://github.com/golang/dep), please go over there and follow their short installation instructions. dep is used to automatically pull in the dependencies for Burrow so you don't have to chase them all down. ### Build and Install ``` $ go get github.com/linkedin/Burrow $ cd $GOPATH/src/github.com/linkedin/Burrow $ dep ensure $ go install ``` ### Running Burrow ``` $ $GOPATH/bin/Burrow --config-dir /path/containing/config ``` ### Using Docker A Docker file is available which builds this project on top of an Alpine Linux image. To use it, build your docker container, mount your Burrow configuration into `/etc/burrow` and run docker. A [Docker Compose](docker-compose.yml) is also available for quick and easy development. Install [Docker Compose](https://docs.docker.com/compose/) and then: 1. Build the docker container: ``` docker-compose build ``` 2. Run the docker compose stack which includes kafka and zookeeper: ``` docker-compose down; docker-compose up ``` 3. Some test topics have already been created by default and Burrow can be accessed on `http://localhost:8000/v3/kafka`. ### Configuration For information on how to write your configuration file, check out the [detailed wiki](https://github.com/linkedin/Burrow/wiki) ## License Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. burrow-1.2.1/config/000077500000000000000000000000001343357346000143105ustar00rootroot00000000000000burrow-1.2.1/config/burrow.toml000066400000000000000000000032071343357346000165270ustar00rootroot00000000000000[general] pidfile="burrow.pid" stdout-logfile="burrow.out" access-control-allow-origin="mysite.example.com" [logging] filename="logs/burrow.log" level="info" maxsize=100 maxbackups=30 maxage=10 use-localtime=false use-compression=true [zookeeper] servers=[ "zkhost01.example.com:2181", "zkhost02.example.com:2181", "zkhost03.example.com:2181" ] timeout=6 root-path="/burrow" [client-profile.test] client-id="burrow-test" kafka-version="0.10.0" [cluster.local] class-name="kafka" servers=[ "kafka01.example.com:10251", "kafka02.example.com:10251", "kafka03.example.com:10251" ] client-profile="test" topic-refresh=120 offset-refresh=30 [consumer.local] class-name="kafka" cluster="local" servers=[ "kafka01.example.com:10251", "kafka02.example.com:10251", "kafka03.example.com:10251" ] client-profile="test" group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$" group-whitelist="" [consumer.local_zk] class-name="kafka_zk" cluster="local" servers=[ "zk01.example.com:2181", "zk02.example.com:2181", "zk03.example.com:2181" ] zookeeper-path="/kafka-cluster" zookeeper-timeout=30 group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$" group-whitelist="" [httpserver.default] address=":8000" [storage.default] class-name="inmemory" workers=20 intervals=15 expire-group=604800 min-distance=1 [notifier.default] class-name="http" url-open="http://someservice.example.com:1467/v1/event" interval=60 timeout=5 keepalive=30 extras={ api_key="REDACTED", app="burrow", tier="STG", fabric="mydc" } template-open="conf/default-http-post.tmpl" template-close="conf/default-http-delete.tmpl" method-close="DELETE" send-close=true threshold=1 burrow-1.2.1/config/default-email.tmpl000066400000000000000000000011111343357346000177110ustar00rootroot00000000000000Subject: [Burrow] Kafka Consumer Lag Alert The Kafka consumer groups you are monitoring are currently showing problems. The following groups are in a problem state (groups not listed are OK): Cluster: {{.Result.Cluster}} Group: {{.Result.Group}} Status: {{.Result.Status.String}} Complete: {{.Result.Complete}} Errors: {{len .Result.Partitions}} partitions have problems {{range .Result.Partitions}} {{.Status.String}} {{.Topic}}:{{.Partition}} ({{.Start.Timestamp}}, {{.Start.Offset}}, {{.Start.Lag}}) -> ({{.End.Timestamp}}, {{.End.Offset}}, {{.End.Lag}}) {{end}} burrow-1.2.1/config/default-http-delete.tmpl000066400000000000000000000001521343357346000210450ustar00rootroot00000000000000{"api_key":"{{index .Extras "api_key"}}","app":"{{index .Extras "app"}}","block":false,"ids":["{{.Id}}"]} burrow-1.2.1/config/default-http-post.tmpl000066400000000000000000000006041343357346000205720ustar00rootroot00000000000000{"api_key":"{{index .Extras "api_key"}}","app":"{{index .Extras "app"}}","block":false,"events":[{"id":"{{.Id}}","event":{"severity":"{{if eq .Result.Status 2}}WARN{{else}}ERR{{end}}","tier":"{{index .Extras "tier"}}","group":"{{.Result.Group}}","start":"{{.Start.Format "Jan 02, 2006 15:04:05 UTC"}}","complete":{{.Result.Complete}},"partitions":{{.Result.Partitions | jsonencoder}}}}]} burrow-1.2.1/config/default-slack-delete.tmpl000066400000000000000000000005761343357346000211750ustar00rootroot00000000000000{ "attachments": [{"color": "good","title": "A kafka consumer is no longer lagging","fields": [{"title": "Group","value": "{{ .Group }}", "short": false},{"title": "Cluster","value": "{{ .Cluster }}","short": true},{"title": "Total Lag","value": "{{ .Result.TotalLag}}","short": true}, {"title": "Start","value": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}","short": true}]}]} burrow-1.2.1/config/default-slack-post.tmpl000066400000000000000000000005761343357346000207200ustar00rootroot00000000000000{ "attachments": [{"color": "danger","title": "A kafka consumer is lagging behind!","fields": [{"title": "Group","value": "{{ .Group }}", "short": false},{"title": "Cluster","value": "{{ .Cluster }}","short": true},{"title": "Total Lag","value": "{{ .Result.TotalLag}}","short": true}, {"title": "Start","value": "{{ .Start.Format "2006-01-02T15:04:05Z07:00" }}","short": true}]}]} burrow-1.2.1/core/000077500000000000000000000000001343357346000137735ustar00rootroot00000000000000burrow-1.2.1/core/burrow.go000066400000000000000000000147221343357346000156500ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package core - Core Burrow logic. // The core package is where all the internal logic for Burrow is located. It provides several helpers for setting up // logging and application management (such as PID files), as well as the Start method that runs Burrow itself. // // The documentation for the rest of the internals, including all the available modules, is available at // https://godoc.org/github.com/linkedin/Burrow/core/internal/?m=all. For the most part, end users of Burrow should not // need to refer to this documentation, as it is targeted at developers of Burrow modules. Details on what modules are // available and how to configure them are available at https://github.com/linkedin/Burrow/wiki package core import ( "os" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/cluster" "github.com/linkedin/Burrow/core/internal/consumer" "github.com/linkedin/Burrow/core/internal/evaluator" "github.com/linkedin/Burrow/core/internal/httpserver" "github.com/linkedin/Burrow/core/internal/notifier" "github.com/linkedin/Burrow/core/internal/storage" "github.com/linkedin/Burrow/core/internal/zookeeper" "github.com/linkedin/Burrow/core/protocol" ) func newCoordinators(app *protocol.ApplicationContext) [7]protocol.Coordinator { // This order is important - it makes sure that the things taking requests start up before things sending requests return [7]protocol.Coordinator{ &zookeeper.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "zookeeper"), ), }, &storage.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "storage"), ), }, &evaluator.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "evaluator"), ), }, &httpserver.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "httpserver"), ), }, ¬ifier.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "notifier"), ), }, &cluster.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "cluster"), ), }, &consumer.Coordinator{ App: app, Log: app.Logger.With( zap.String("type", "coordinator"), zap.String("name", "consumer"), ), }, } } func configureCoordinators(app *protocol.ApplicationContext, coordinators [7]protocol.Coordinator) { // Configure methods are allowed to panic, as their errors are non-recoverable // Catch panics here and flag in the application context if we can't continue defer func() { if r := recover(); r != nil { app.Logger.Panic(r.(string)) app.ConfigurationValid = false } }() // Configure the coordinators in order for _, coordinator := range coordinators { coordinator.Configure() } app.ConfigurationValid = true } // Start is called to start the Burrow application. This is exposed so that it is possible to use Burrow as a library // from within another application. Prior to calling this func, the configuration must have been loaded by viper from // some underlying source (e.g. a TOML configuration file, or explicitly set in code after reading from another source). // This func will block upon being called. // // If the calling application would like to control logging, it can pass a pointer to an instantiated // protocol.ApplicationContext struct that has the Logger and LogLevel fields set. Otherwise, Start will create a // logger based on configurations in viper. // // exitChannel is a signal channel that is provided by the calling application in order to signal Burrow to shut down. // Burrow does not currently check the signal type: if any message is received on the channel, or if the channel is // closed, Burrow will exit and Start will return 0. // // Start will return a 1 on any failure, including invalid configurations or a failure to start Burrow modules. func Start(app *protocol.ApplicationContext, exitChannel chan os.Signal) int { // Validate that the ApplicationContext is complete if (app == nil) || (app.Logger == nil) || (app.LogLevel == nil) { // Didn't get a valid ApplicationContext, so we'll set up our own, with the logger app = &protocol.ApplicationContext{} app.Logger, app.LogLevel = ConfigureLogger() defer app.Logger.Sync() } app.Logger.Info("Started Burrow") // Set up a specific child logger for main log := app.Logger.With(zap.String("type", "main"), zap.String("name", "burrow")) // Set up an array of coordinators in the order they are to be loaded (and closed) coordinators := newCoordinators(app) // Set up two main channels to use for the evaluator and storage coordinators. This is how burrow communicates // internally: // * Consumers and Clusters send offsets to the storage coordinator to populate all the state information // * The Notifiers send evaluation requests to the evaluator coordinator to check group status // * The Evaluators send requests to the storage coordinator for group offset and lag information // * The HTTP server sends requests to both the evaluator and storage coordinators to fulfill API requests app.EvaluatorChannel = make(chan *protocol.EvaluatorRequest) app.StorageChannel = make(chan *protocol.StorageRequest) // Configure coordinators and exit if anything fails configureCoordinators(app, coordinators) if !app.ConfigurationValid { return 1 } // Start the coordinators in order for i, coordinator := range coordinators { err := coordinator.Start() if err != nil { // Reverse our way out, stopping coordinators, then exit for j := i - 1; j >= 0; j-- { coordinators[j].Stop() } return 1 } } // Wait until we're told to exit <-exitChannel log.Info("Shutdown triggered") // Stop the coordinators in the reverse order. This assures that request senders are stopped before request servers for i := len(coordinators) - 1; i >= 0; i-- { coordinators[i].Stop() } // Exit cleanly return 0 } burrow-1.2.1/core/internal/000077500000000000000000000000001343357346000156075ustar00rootroot00000000000000burrow-1.2.1/core/internal/cluster/000077500000000000000000000000001343357346000172705ustar00rootroot00000000000000burrow-1.2.1/core/internal/cluster/coordinator.go000066400000000000000000000106621343357346000221470ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package cluster - Kafka cluster subsystem. // The cluster subsystem is responsible for getting topic and partition information, as well as current broker offsets, // from Kafka clusters and sending that information to the storage subsystem. It does not handle any consumer group // information. // // Modules // // Currently, the following modules are provided: // // * kafka - Fetch topic, partition, and offset information from a Kafka cluster package cluster import ( "errors" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // A "cluster" is a single Kafka cluster that is going to be monitored by Burrow. The cluster module is responsible for // connecting to the Kafka cluster, monitoring the topic list, and periodically fetching the broker end offset (latest // offset) for each partition. This information is sent to the storage subsystem, where it can be retrieved by the // evaluator and HTTP server. // Coordinator manages all cluster modules, making sure they are configured, started, and stopped at the appropriate // time. type Coordinator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger modules map[string]protocol.Module } // getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there // is any error, it will panic with an appropriate message describing the problem. func getModuleForClass(app *protocol.ApplicationContext, moduleName string, className string) protocol.Module { switch className { case "kafka": return &KafkaCluster{ App: app, Log: app.Logger.With( zap.String("type", "module"), zap.String("coordinator", "cluster"), zap.String("class", className), zap.String("name", moduleName), ), } default: panic("Unknown cluster className provided: " + className) } } // Configure is called to create each of the configured cluster modules and call their Configure funcs to validate // their individual configurations and set them up. If there are any problems, it is expected that these funcs will // panic with a descriptive error message, as configuration failures are not recoverable errors. func (bc *Coordinator) Configure() { bc.Log.Info("configuring") bc.modules = make(map[string]protocol.Module) // Create all configured cluster modules, add to list of clusters modules := viper.GetStringMap("cluster") for name := range modules { configRoot := "cluster." + name module := getModuleForClass(bc.App, name, viper.GetString(configRoot+".class-name")) module.Configure(name, configRoot) bc.modules[name] = module } } // Start calls each of the configured cluster modules' underlying Start funcs. As the coordinator itself has no ongoing // work to do, it does not start any other goroutines. If any module Start returns an error, this func stops immediately // and returns that error to the caller. No further modules will be loaded after that. func (bc *Coordinator) Start() error { bc.Log.Info("starting") // Start Cluster modules err := helpers.StartCoordinatorModules(bc.modules) if err != nil { return errors.New("Error starting cluster module: " + err.Error()) } return nil } // Stop calls each of the configured cluster modules' underlying Stop funcs. It is expected that the module Stop will // not return until the module has been completely stopped. While an error can be returned, this func always returns no // error, as a failure during stopping is not a critical failure func (bc *Coordinator) Stop() error { bc.Log.Info("stopping") // The individual cluster modules can choose whether or not to implement a wait in the Stop routine helpers.StopCoordinatorModules(bc.modules) return nil } burrow-1.2.1/core/internal/cluster/coordinator_test.go000066400000000000000000000042161343357346000232040ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package cluster import ( "testing" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "go.uber.org/zap" ) func fixtureCoordinator() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("client-profile..client-id", "testid") viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) return &coordinator } func TestCoordinator_ImplementsCoordinator(t *testing.T) { assert.Implements(t, (*protocol.Coordinator)(nil), new(Coordinator)) } func TestCoordinator_Configure(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) } func TestCoordinator_Configure_TwoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("cluster.anothertest.class-name", "kafka") viper.Set("cluster.anothertest.servers", []string{"broker1.example.com:1234"}) coordinator.Configure() } func TestCoordinator_StartStop(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // Swap out the coordinator modules with a mock for testing mockModule := &helpers.MockModule{} mockModule.On("Start").Return(nil) mockModule.On("Stop").Return(nil) coordinator.modules["test"] = mockModule coordinator.Start() mockModule.AssertCalled(t, "Start") coordinator.Stop() mockModule.AssertCalled(t, "Stop") } burrow-1.2.1/core/internal/cluster/kafka_cluster.go000066400000000000000000000220261343357346000224370ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package cluster import ( "sync" "time" "github.com/Shopify/sarama" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // KafkaCluster is a cluster module which connects to a single Apache Kafka cluster and manages the broker topic and // partition information. It periodically updates a list of all topics and partitions, and also fetches the broker // end offset (latest) for each partition. This information is forwarded to the storage module for use in consumer // evaluations. type KafkaCluster struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string saramaConfig *sarama.Config servers []string offsetRefresh int topicRefresh int offsetTicker *time.Ticker metadataTicker *time.Ticker quitChannel chan struct{} running sync.WaitGroup fetchMetadata bool topicMap map[string]int } // Configure validates the configuration for the cluster. At minimum, there must be a list of servers provided for the // Kafka cluster, of the form host:port. Default values will be set for the intervals to use for refreshing offsets // (10 seconds) and topics (60 seconds). A missing, or bad, list of servers will cause this func to panic. func (module *KafkaCluster) Configure(name string, configRoot string) { module.Log.Info("configuring") module.name = name module.quitChannel = make(chan struct{}) module.running = sync.WaitGroup{} profile := viper.GetString(configRoot + ".client-profile") module.saramaConfig = helpers.GetSaramaConfigFromClientProfile(profile) module.servers = viper.GetStringSlice(configRoot + ".servers") if len(module.servers) == 0 { panic("No Kafka brokers specified for cluster " + module.name) } else if !helpers.ValidateHostList(module.servers) { panic("Cluster '" + name + "' has one or more improperly formatted servers (must be host:port)") } // Set defaults for configs if needed viper.SetDefault(configRoot+".offset-refresh", 10) viper.SetDefault(configRoot+".topic-refresh", 60) module.offsetRefresh = viper.GetInt(configRoot + ".offset-refresh") module.topicRefresh = viper.GetInt(configRoot + ".topic-refresh") } // Start connects to the Kafka cluster using the Shopify/sarama client. Any error connecting to the cluster is returned // to the caller. Once the client is set up, tickers are started to periodically refresh topics and offsets. func (module *KafkaCluster) Start() error { module.Log.Info("starting") // Connect Kafka client client, err := sarama.NewClient(module.servers, module.saramaConfig) if err != nil { module.Log.Error("failed to start client", zap.Error(err)) return err } // Fire off the offset requests once, before we start the ticker, to make sure we start with good data for consumers helperClient := &helpers.BurrowSaramaClient{ Client: client, } module.fetchMetadata = true module.getOffsets(helperClient) // Start main loop that has a timer for offset and topic fetches module.offsetTicker = time.NewTicker(time.Duration(module.offsetRefresh) * time.Second) module.metadataTicker = time.NewTicker(time.Duration(module.topicRefresh) * time.Second) go module.mainLoop(helperClient) return nil } // Stop causes both the topic and offset refresh tickers to be stopped, and then it closes the Kafka client. func (module *KafkaCluster) Stop() error { module.Log.Info("stopping") module.metadataTicker.Stop() module.offsetTicker.Stop() close(module.quitChannel) module.running.Wait() return nil } func (module *KafkaCluster) mainLoop(client helpers.SaramaClient) { module.running.Add(1) defer module.running.Done() for { select { case <-module.offsetTicker.C: module.getOffsets(client) case <-module.metadataTicker.C: // Update metadata on next offset fetch module.fetchMetadata = true case <-module.quitChannel: return } } } func (module *KafkaCluster) maybeUpdateMetadataAndDeleteTopics(client helpers.SaramaClient) { if module.fetchMetadata { module.fetchMetadata = false client.RefreshMetadata() // Get the current list of topics and make a map topicList, err := client.Topics() if err != nil { module.Log.Error("failed to fetch topic list", zap.String("sarama_error", err.Error())) return } // We'll use the partition counts later topicMap := make(map[string]int) for _, topic := range topicList { partitions, err := client.Partitions(topic) if err != nil { module.Log.Error("failed to fetch partition list", zap.String("sarama_error", err.Error())) return } topicMap[topic] = len(partitions) } // Check for deleted topics if we have a previous map to check against if module.topicMap != nil { for topic := range module.topicMap { if _, ok := topicMap[topic]; !ok { // Topic no longer exists - tell storage to delete it module.App.StorageChannel <- &protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteTopic, Cluster: module.name, Topic: topic, } } } } // Save the new topicMap for next time module.topicMap = topicMap } } func (module *KafkaCluster) generateOffsetRequests(client helpers.SaramaClient) (map[int32]*sarama.OffsetRequest, map[int32]helpers.SaramaBroker) { requests := make(map[int32]*sarama.OffsetRequest) brokers := make(map[int32]helpers.SaramaBroker) // Generate an OffsetRequest for each topic:partition and bucket it to the leader broker errorTopics := make(map[string]bool) for topic, partitions := range module.topicMap { for i := 0; i < partitions; i++ { broker, err := client.Leader(topic, int32(i)) if err != nil { module.Log.Warn("failed to fetch leader for partition", zap.String("topic", topic), zap.Int("partition", i), zap.String("sarama_error", err.Error())) errorTopics[topic] = true continue } if _, ok := requests[broker.ID()]; !ok { requests[broker.ID()] = &sarama.OffsetRequest{} } brokers[broker.ID()] = broker requests[broker.ID()].AddBlock(topic, int32(i), sarama.OffsetNewest, 1) } } // If there are any topics that had errors, force a metadata refresh on the next run if len(errorTopics) > 0 { module.fetchMetadata = true } return requests, brokers } // This function performs massively parallel OffsetRequests, which is better than Sarama's internal implementation, // which does one at a time. Several orders of magnitude faster. func (module *KafkaCluster) getOffsets(client helpers.SaramaClient) { module.maybeUpdateMetadataAndDeleteTopics(client) requests, brokers := module.generateOffsetRequests(client) // Send out the OffsetRequest to each broker for all the partitions it is leader for // The results go to the offset storage module var wg = sync.WaitGroup{} var errorTopics = sync.Map{} getBrokerOffsets := func(brokerID int32, request *sarama.OffsetRequest) { defer wg.Done() response, err := brokers[brokerID].GetAvailableOffsets(request) if err != nil { module.Log.Error("failed to fetch offsets from broker", zap.String("sarama_error", err.Error()), zap.Int32("broker", brokerID), ) brokers[brokerID].Close() return } ts := time.Now().Unix() * 1000 for topic, partitions := range response.Blocks { for partition, offsetResponse := range partitions { if offsetResponse.Err != sarama.ErrNoError { module.Log.Warn("error in OffsetResponse", zap.String("sarama_error", offsetResponse.Err.Error()), zap.Int32("broker", brokerID), zap.String("topic", topic), zap.Int32("partition", partition), ) // Gather a list of topics that had errors errorTopics.Store(topic, true) continue } offset := &protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: module.name, Topic: topic, Partition: partition, Offset: offsetResponse.Offsets[0], Timestamp: ts, TopicPartitionCount: int32(module.topicMap[topic]), } helpers.TimeoutSendStorageRequest(module.App.StorageChannel, offset, 1) } } } for brokerID, request := range requests { wg.Add(1) go getBrokerOffsets(brokerID, request) } wg.Wait() // If there are any topics that had errors, force a metadata refresh on the next run errorTopics.Range(func(key, value interface{}) bool { module.fetchMetadata = true return false }) } burrow-1.2.1/core/internal/cluster/kafka_cluster_test.go000066400000000000000000000227351343357346000235050ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package cluster import ( "errors" "testing" "time" "github.com/Shopify/sarama" "github.com/spf13/viper" "go.uber.org/zap" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" "sync" ) func fixtureModule() *KafkaCluster { module := KafkaCluster{ Log: zap.NewNop(), } module.App = &protocol.ApplicationContext{ StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("client-profile..client-id", "testid") viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) return &module } func TestKafkaCluster_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(KafkaCluster)) } func TestKafkaCluster_Configure(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") assert.NotNil(t, module.saramaConfig, "Expected saramaConfig to be populated") } func TestKafkaCluster_Configure_DefaultIntervals(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") assert.Equal(t, int(10), module.offsetRefresh, "Default OffsetRefresh value of 10 did not get set") assert.Equal(t, int(60), module.topicRefresh, "Default TopicRefresh value of 60 did not get set") } func TestKafkaCluster_maybeUpdateMetadataAndDeleteTopics_NoUpdate(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") client := &helpers.MockSaramaClient{} module.maybeUpdateMetadataAndDeleteTopics(client) client.AssertNotCalled(t, "RefreshMetadata") } func TestKafkaCluster_maybeUpdateMetadataAndDeleteTopics_NoDelete(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") // Set up the mock to return a test topic and partition client := &helpers.MockSaramaClient{} client.On("RefreshMetadata").Return(nil) client.On("Topics").Return([]string{"testtopic"}, nil) client.On("Partitions", "testtopic").Return([]int32{0}, nil) module.fetchMetadata = true module.maybeUpdateMetadataAndDeleteTopics(client) client.AssertExpectations(t) assert.False(t, module.fetchMetadata, "Expected fetchMetadata to be reset to false") assert.Lenf(t, module.topicMap, 1, "Expected 1 topic entry, not %v", len(module.topicMap)) topic, ok := module.topicMap["testtopic"] assert.True(t, ok, "Expected to find testtopic in topicMap") assert.Equalf(t, 1, topic, "Expected testtopic to be recorded with 1 partition, not %v", topic) } func TestKafkaCluster_maybeUpdateMetadataAndDeleteTopics_Delete(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") // Set up the mock to return a test topic and partition client := &helpers.MockSaramaClient{} client.On("RefreshMetadata").Return(nil) client.On("Topics").Return([]string{"testtopic"}, nil) client.On("Partitions", "testtopic").Return([]int32{0}, nil) module.fetchMetadata = true module.topicMap = make(map[string]int) module.topicMap["topictodelete"] = 10 // Need to wait for this request to come in and finish, which happens when we call maybeUpdate... wg := &sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() request := <-module.App.StorageChannel assert.Equalf(t, protocol.StorageSetDeleteTopic, request.RequestType, "Expected request sent with type StorageSetDeleteTopic, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "topictodelete", request.Topic, "Expected request sent with topic topictodelete, not %v", request.Topic) }() module.maybeUpdateMetadataAndDeleteTopics(client) wg.Wait() client.AssertExpectations(t) assert.False(t, module.fetchMetadata, "Expected fetchMetadata to be reset to false") assert.Lenf(t, module.topicMap, 1, "Expected 1 topic entry, not %v", len(module.topicMap)) topic, ok := module.topicMap["testtopic"] assert.True(t, ok, "Expected to find testtopic in topicMap") assert.Equalf(t, 1, topic, "Expected testtopic to be recorded with 1 partition, not %v", topic) } func TestKafkaCluster_generateOffsetRequests(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") module.topicMap = make(map[string]int) module.topicMap["testtopic"] = 1 // Set up a broker mock broker := &helpers.MockSaramaBroker{} broker.On("ID").Return(int32(13)) // Set up the mock to return the leader broker for a test topic and partition client := &helpers.MockSaramaClient{} client.On("Leader", "testtopic", int32(0)).Return(broker, nil) requests, brokers := module.generateOffsetRequests(client) broker.AssertExpectations(t) client.AssertExpectations(t) assert.Lenf(t, brokers, 1, "Expected 1 broker entry, not %v", len(brokers)) _, ok := brokers[13] assert.True(t, ok, "Expected key for the broker to be its ID") assert.Equal(t, broker, brokers[13], "Expected broker returned to be the mock") assert.Lenf(t, requests, 1, "Expected 1 request, not %v", len(requests)) } func TestKafkaCluster_generateOffsetRequests_NoLeader(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") module.topicMap = make(map[string]int) module.topicMap["testtopic"] = 2 // Set up a broker mock broker := &helpers.MockSaramaBroker{} broker.On("ID").Return(int32(13)) // Set up the mock to return the leader broker for a test topic and partition client := &helpers.MockSaramaClient{} var nilBroker *helpers.BurrowSaramaBroker client.On("Leader", "testtopic", int32(0)).Return(nilBroker, errors.New("no leader error")) client.On("Leader", "testtopic", int32(1)).Return(broker, nil) requests, brokers := module.generateOffsetRequests(client) broker.AssertExpectations(t) client.AssertExpectations(t) assert.Lenf(t, brokers, 1, "Expected 1 broker entry, not %v", len(brokers)) _, ok := brokers[13] assert.True(t, ok, "Expected key for the broker to be its ID") assert.Equal(t, broker, brokers[13], "Expected broker returned to be the mock") assert.Lenf(t, requests, 1, "Expected 1 request, not %v", len(requests)) assert.True(t, module.fetchMetadata, "Expected fetchMetadata to be true") } func TestKafkaCluster_getOffsets(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") module.topicMap = make(map[string]int) module.topicMap["testtopic"] = 2 module.fetchMetadata = false // Set up an OffsetResponse offsetResponse := &sarama.OffsetResponse{Version: 1} offsetResponse.AddTopicPartition("testtopic", 0, 8374) // Set up a broker mock broker := &helpers.MockSaramaBroker{} broker.On("ID").Return(int32(13)) broker.On("GetAvailableOffsets", mock.MatchedBy(func(request *sarama.OffsetRequest) bool { return request != nil })).Return(offsetResponse, nil) // Set up the mock to return the leader broker for a test topic and partition client := &helpers.MockSaramaClient{} var nilBroker *helpers.BurrowSaramaBroker client.On("Leader", "testtopic", int32(0)).Return(broker, nil) client.On("Leader", "testtopic", int32(1)).Return(nilBroker, errors.New("no leader error")) go module.getOffsets(client) request := <-module.App.StorageChannel broker.AssertExpectations(t) client.AssertExpectations(t) assert.Equalf(t, protocol.StorageSetBrokerOffset, request.RequestType, "Expected request sent with type StorageSetBrokerOffset, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "testtopic", request.Topic, "Expected request sent with topic testtopic, not %v", request.Topic) assert.Equalf(t, int32(0), request.Partition, "Expected request sent with partition 0, not %v", request.Partition) assert.Equalf(t, int32(2), request.TopicPartitionCount, "Expected request sent with TopicPartitionCount 2, not %v", request.TopicPartitionCount) assert.Equalf(t, int64(8374), request.Offset, "Expected request sent with offset 8374, not %v", request.Offset) assert.True(t, module.fetchMetadata, "Expected fetchMetadata to be true") // Make sure there is nothing else on the channel time.Sleep(100 * time.Millisecond) select { case <-module.App.StorageChannel: t.Fatal("Expected no additional value waiting on storage channel") default: break } } func TestKafkaCluster_getOffsets_BrokerFailed(t *testing.T) { module := fixtureModule() module.Configure("test", "cluster.test") module.topicMap = make(map[string]int) module.topicMap["testtopic"] = 1 module.fetchMetadata = false // Set up a broker mock broker := &helpers.MockSaramaBroker{} broker.On("ID").Return(int32(13)) var offsetResponse *sarama.OffsetResponse broker.On("GetAvailableOffsets", mock.MatchedBy(func(request *sarama.OffsetRequest) bool { return request != nil })).Return(offsetResponse, errors.New("broker failed")) broker.On("Close").Return(nil) // Set up the mock to return the leader broker for a test topic and partition client := &helpers.MockSaramaClient{} client.On("Leader", "testtopic", int32(0)).Return(broker, nil) module.getOffsets(client) broker.AssertExpectations(t) client.AssertExpectations(t) } burrow-1.2.1/core/internal/consumer/000077500000000000000000000000001343357346000174425ustar00rootroot00000000000000burrow-1.2.1/core/internal/consumer/coordinator.go000066400000000000000000000113141343357346000223140ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package consumer - Kafka consumer subsystem. // The consumer subsystem is responsible for getting consumer offset information and sending that information to the // storage subsystem. This consumer information could be stored in a variety of places, and each module supports a // different type of repository. // // Modules // // Currently, the following modules are provided: // // * kafka - Consume a Kafka cluster's __consumer_offsets topic to get consumer information (new consumer) // // * kafka_zk - Parse the /consumers tree of a Kafka cluster's metadata to get consumer information (old consumer) package consumer import ( "errors" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // The consumer module is responsible for fetching information about consumer group status from some external system // and forwarding it to the storage module. Each consumer module is associated with a single cluster. // Coordinator manages all consumer modules, making sure they are configured, started, and stopped at the appropriate // time. type Coordinator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger modules map[string]protocol.Module } // getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there // is any error, it will panic with an appropriate message describing the problem. func getModuleForClass(app *protocol.ApplicationContext, moduleName string, className string) protocol.Module { logger := app.Logger.With( zap.String("type", "module"), zap.String("coordinator", "consumer"), zap.String("class", className), zap.String("name", moduleName), ) switch className { case "kafka": return &KafkaClient{ App: app, Log: logger, } case "kafka_zk": return &KafkaZkClient{ App: app, Log: logger, } default: panic("Unknown consumer className provided: " + className) } } // Configure is called to create each of the configured consumer modules and call their Configure funcs to validate // their individual configurations and set them up. If there are any problems, it is expected that these funcs will // panic with a descriptive error message, as configuration failures are not recoverable errors. func (cc *Coordinator) Configure() { cc.Log.Info("configuring") cc.modules = make(map[string]protocol.Module) // Create all configured cluster modules, add to list of clusters modules := viper.GetStringMap("consumer") for name := range modules { configRoot := "consumer." + name if !viper.IsSet("cluster." + viper.GetString(configRoot+".cluster")) { panic("Consumer '" + name + "' references an unknown cluster '" + viper.GetString(configRoot+".cluster") + "'") } module := getModuleForClass(cc.App, name, viper.GetString(configRoot+".class-name")) module.Configure(name, configRoot) cc.modules[name] = module } } // Start calls each of the configured consumer modules' underlying Start funcs. As the coordinator itself has no ongoing // work to do, it does not start any other goroutines. If any module Start returns an error, this func stops immediately // and returns that error to the caller. No further modules will be loaded after that. func (cc *Coordinator) Start() error { cc.Log.Info("starting") // Start Consumer modules err := helpers.StartCoordinatorModules(cc.modules) if err != nil { return errors.New("Error starting consumer module: " + err.Error()) } return nil } // Stop calls each of the configured consumer modules' underlying Stop funcs. It is expected that the module Stop will // not return until the module has been completely stopped. While an error can be returned, this func always returns no // error, as a failure during stopping is not a critical failure func (cc *Coordinator) Stop() error { cc.Log.Info("stopping") // The individual consumer modules can choose whether or not to implement a wait in the Stop routine helpers.StopCoordinatorModules(cc.modules) return nil } burrow-1.2.1/core/internal/consumer/coordinator_test.go000066400000000000000000000050711343357346000233560ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package consumer import ( "github.com/stretchr/testify/assert" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) func fixtureCoordinator() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("client-profile..client-id", "testid") viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.test.class-name", "kafka") viper.Set("consumer.test.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.test.cluster", "test") return &coordinator } func TestCoordinator_ImplementsCoordinator(t *testing.T) { assert.Implements(t, (*protocol.Coordinator)(nil), new(Coordinator)) } func TestCoordinator_Configure(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) } func TestCoordinator_Configure_BadCluster(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("consumer.test.cluster", "nocluster") assert.Panics(t, coordinator.Configure, "Expected panic") } func TestCoordinator_Configure_TwoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("consumer.anothertest.class-name", "kafka") viper.Set("consumer.anothertest.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.anothertest.cluster", "test") coordinator.Configure() } func TestCoordinator_StartStop(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // Swap out the coordinator modules with a mock for testing mockModule := &helpers.MockModule{} mockModule.On("Start").Return(nil) mockModule.On("Stop").Return(nil) coordinator.modules["test"] = mockModule coordinator.Start() mockModule.AssertCalled(t, "Start") coordinator.Stop() mockModule.AssertCalled(t, "Stop") } burrow-1.2.1/core/internal/consumer/kafka_client.go000066400000000000000000000445541343357346000224200ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package consumer import ( "bytes" "encoding/binary" "errors" "sync" "github.com/Shopify/sarama" "go.uber.org/zap" "regexp" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" "github.com/spf13/viper" ) // KafkaClient is a consumer module which connects to a single Apache Kafka cluster and reads consumer group information // from the offsets topic in the cluster, which is typically __consumer_offsets. The messages in this topic are decoded // and the information is forwarded to the storage subsystem for use in evaluations. type KafkaClient struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string cluster string servers []string offsetsTopic string startLatest bool saramaConfig *sarama.Config groupWhitelist *regexp.Regexp groupBlacklist *regexp.Regexp quitChannel chan struct{} running sync.WaitGroup } type offsetKey struct { Group string Topic string Partition int32 ErrorAt string } type offsetValue struct { Offset int64 Timestamp int64 ErrorAt string } type metadataHeader struct { ProtocolType string Generation int32 Protocol string Leader string } type metadataMember struct { MemberID string ClientID string ClientHost string RebalanceTimeout int32 SessionTimeout int32 Assignment map[string][]int32 } // Configure validates the configuration for the consumer. At minimum, there must be a cluster name to which these // consumers belong, as well as a list of servers provided for the Kafka cluster, of the form host:port. If not // explicitly configured, the offsets topic is set to the default for Kafka, which is __consumer_offsets. If the // cluster name is unknown, or if the server list is missing or invalid, this func will panic. func (module *KafkaClient) Configure(name string, configRoot string) { module.Log.Info("configuring") module.name = name module.quitChannel = make(chan struct{}) module.running = sync.WaitGroup{} module.cluster = viper.GetString(configRoot + ".cluster") if !viper.IsSet("cluster." + module.cluster) { panic("Consumer '" + name + "' references an unknown cluster '" + module.cluster + "'") } profile := viper.GetString(configRoot + ".client-profile") module.saramaConfig = helpers.GetSaramaConfigFromClientProfile(profile) module.servers = viper.GetStringSlice(configRoot + ".servers") if len(module.servers) == 0 { panic("No Kafka brokers specified for consumer " + module.name) } else if !helpers.ValidateHostList(module.servers) { panic("Consumer '" + name + "' has one or more improperly formatted servers (must be host:port)") } // Set defaults for configs if needed, and get them viper.SetDefault(configRoot+".offsets-topic", "__consumer_offsets") module.offsetsTopic = viper.GetString(configRoot + ".offsets-topic") module.startLatest = viper.GetBool(configRoot + ".start-latest") whitelist := viper.GetString(configRoot + ".group-whitelist") if whitelist != "" { re, err := regexp.Compile(whitelist) if err != nil { module.Log.Panic("Failed to compile group whitelist") panic(err) } module.groupWhitelist = re } blacklist := viper.GetString(configRoot + ".group-blacklist") if blacklist != "" { re, err := regexp.Compile(blacklist) if err != nil { module.Log.Panic("Failed to compile group blacklist") panic(err) } module.groupBlacklist = re } } // Start connects to the Kafka cluster using the Shopify/sarama client. Any error connecting to the cluster is returned // to the caller. Once the client is set up, the consumers for the configured offsets topic are started. func (module *KafkaClient) Start() error { module.Log.Info("starting") // Connect Kafka client client, err := sarama.NewClient(module.servers, module.saramaConfig) if err != nil { module.Log.Error("failed to start client", zap.Error(err)) return err } // Start the consumers err = module.startKafkaConsumer(&helpers.BurrowSaramaClient{Client: client}) if err != nil { module.Log.Error("failed to start consumer", zap.Error(err)) client.Close() return err } return nil } // Stop closes the goroutines that listen to the client consumer. func (module *KafkaClient) Stop() error { module.Log.Info("stopping") close(module.quitChannel) module.running.Wait() return nil } func (module *KafkaClient) partitionConsumer(consumer sarama.PartitionConsumer) { defer module.running.Done() defer consumer.AsyncClose() for { select { case msg := <-consumer.Messages(): module.processConsumerOffsetsMessage(msg) case err := <-consumer.Errors(): module.Log.Error("consume error", zap.String("topic", err.Topic), zap.Int32("partition", err.Partition), zap.String("error", err.Err.Error()), ) case <-module.quitChannel: return } } } func (module *KafkaClient) startKafkaConsumer(client helpers.SaramaClient) error { // Create the consumer from the client consumer, err := client.NewConsumerFromClient() if err != nil { module.Log.Error("failed to get new consumer", zap.Error(err)) client.Close() return err } // Get a partition count for the consumption topic partitions, err := client.Partitions(module.offsetsTopic) if err != nil { module.Log.Error("failed to get partition count", zap.String("topic", module.offsetsTopic), zap.String("error", err.Error()), ) client.Close() return err } // Default to bootstrapping the offsets topic, unless configured otherwise startFrom := sarama.OffsetOldest if module.startLatest { startFrom = sarama.OffsetNewest } // Start consumers for each partition with fan in module.Log.Info("starting consumers", zap.String("topic", module.offsetsTopic), zap.Int("count", len(partitions)), ) for i, partition := range partitions { pconsumer, err := consumer.ConsumePartition(module.offsetsTopic, partition, startFrom) if err != nil { module.Log.Error("failed to consume partition", zap.String("topic", module.offsetsTopic), zap.Int("partition", i), zap.String("error", err.Error()), ) return err } module.running.Add(1) go module.partitionConsumer(pconsumer) } return nil } func (module *KafkaClient) processConsumerOffsetsMessage(msg *sarama.ConsumerMessage) { logger := module.Log.With( zap.String("offset_topic", msg.Topic), zap.Int32("offset_partition", msg.Partition), zap.Int64("offset_offset", msg.Offset), ) if len(msg.Value) == 0 { // Tombstone message - we don't handle them for now logger.Debug("dropped tombstone") return } var keyver int16 keyBuffer := bytes.NewBuffer(msg.Key) err := binary.Read(keyBuffer, binary.BigEndian, &keyver) if err != nil { logger.Warn("failed to decode", zap.String("reason", "no key version"), ) return } switch keyver { case 0, 1: module.decodeKeyAndOffset(keyBuffer, msg.Value, logger) case 2: module.decodeGroupMetadata(keyBuffer, msg.Value, logger) default: logger.Warn("failed to decode", zap.String("reason", "key version"), zap.Int16("version", keyver), ) } } func readString(buf *bytes.Buffer) (string, error) { var strlen int16 err := binary.Read(buf, binary.BigEndian, &strlen) if err != nil { return "", err } if strlen == -1 { return "", nil } strbytes := make([]byte, strlen) n, err := buf.Read(strbytes) if (err != nil) || (n != int(strlen)) { return "", errors.New("string underflow") } return string(strbytes), nil } func (module *KafkaClient) acceptConsumerGroup(group string) bool { if (module.groupWhitelist != nil) && (!module.groupWhitelist.MatchString(group)) { return false } if (module.groupBlacklist != nil) && module.groupBlacklist.MatchString(group) { return false } return true } func (module *KafkaClient) decodeKeyAndOffset(keyBuffer *bytes.Buffer, value []byte, logger *zap.Logger) { // Version 0 and 1 keys are decoded the same way offsetKey, errorAt := decodeOffsetKeyV0(keyBuffer) if errorAt != "" { logger.Warn("failed to decode", zap.String("message_type", "offset"), zap.String("group", offsetKey.Group), zap.String("topic", offsetKey.Topic), zap.Int32("partition", offsetKey.Partition), zap.String("reason", errorAt), ) return } offsetLogger := logger.With( zap.String("message_type", "offset"), zap.String("group", offsetKey.Group), zap.String("topic", offsetKey.Topic), zap.Int32("partition", offsetKey.Partition), ) if !module.acceptConsumerGroup(offsetKey.Group) { offsetLogger.Debug("dropped", zap.String("reason", "whitelist")) return } var valueVersion int16 valueBuffer := bytes.NewBuffer(value) err := binary.Read(valueBuffer, binary.BigEndian, &valueVersion) if err != nil { offsetLogger.Warn("failed to decode", zap.String("reason", "no value version"), ) return } switch valueVersion { case 0, 1: module.decodeAndSendOffset(offsetKey, valueBuffer, offsetLogger, decodeOffsetValueV0) case 3: module.decodeAndSendOffset(offsetKey, valueBuffer, offsetLogger, decodeOffsetValueV3) default: offsetLogger.Warn("failed to decode", zap.String("reason", "value version"), zap.Int16("version", valueVersion), ) } } func (module *KafkaClient) decodeAndSendOffset(offsetKey offsetKey, valueBuffer *bytes.Buffer, logger *zap.Logger, decoder func(*bytes.Buffer) (offsetValue, string)) { offsetValue, errorAt := decoder(valueBuffer) if errorAt != "" { logger.Warn("failed to decode", zap.Int64("offset", offsetValue.Offset), zap.Int64("timestamp", offsetValue.Timestamp), zap.String("reason", errorAt), ) return } partitionOffset := &protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: module.cluster, Topic: offsetKey.Topic, Partition: int32(offsetKey.Partition), Group: offsetKey.Group, Timestamp: int64(offsetValue.Timestamp), Offset: int64(offsetValue.Offset), } logger.Debug("consumer offset", zap.Int64("offset", offsetValue.Offset), zap.Int64("timestamp", offsetValue.Timestamp), ) helpers.TimeoutSendStorageRequest(module.App.StorageChannel, partitionOffset, 1) } func (module *KafkaClient) decodeGroupMetadata(keyBuffer *bytes.Buffer, value []byte, logger *zap.Logger) { group, err := readString(keyBuffer) if err != nil { logger.Warn("failed to decode", zap.String("message_type", "metadata"), zap.String("reason", "group"), ) return } var valueVersion int16 valueBuffer := bytes.NewBuffer(value) err = binary.Read(valueBuffer, binary.BigEndian, &valueVersion) if err != nil { logger.Warn("failed to decode", zap.String("message_type", "metadata"), zap.String("group", group), zap.String("reason", "no value version"), ) return } switch valueVersion { case 0, 1: module.decodeAndSendGroupMetadata(valueVersion, group, valueBuffer, logger.With( zap.String("message_type", "metadata"), zap.String("group", group), )) default: logger.Warn("failed to decode", zap.String("message_type", "metadata"), zap.String("group", group), zap.String("reason", "value version"), zap.Int16("version", valueVersion), ) } } func (module *KafkaClient) decodeAndSendGroupMetadata(valueVersion int16, group string, valueBuffer *bytes.Buffer, logger *zap.Logger) { metadataHeader, errorAt := decodeMetadataValueHeader(valueBuffer) metadataLogger := logger.With( zap.String("protocol_type", metadataHeader.ProtocolType), zap.Int32("generation", metadataHeader.Generation), zap.String("protocol", metadataHeader.Protocol), zap.String("leader", metadataHeader.Leader), ) if errorAt != "" { metadataLogger.Warn("failed to decode", zap.String("reason", errorAt), ) return } var memberCount int32 err := binary.Read(valueBuffer, binary.BigEndian, &memberCount) if err != nil { metadataLogger.Warn("failed to decode", zap.String("reason", "no member size"), ) return } // If memberCount is zero, clear all ownership if memberCount == 0 { metadataLogger.Debug("clear owners") helpers.TimeoutSendStorageRequest(module.App.StorageChannel, &protocol.StorageRequest{ RequestType: protocol.StorageClearConsumerOwners, Cluster: module.cluster, Group: group, }, 1) return } count := int(memberCount) for i := 0; i < count; i++ { member, errorAt := decodeMetadataMember(valueBuffer, valueVersion) if errorAt != "" { metadataLogger.Warn("failed to decode", zap.String("reason", errorAt), ) return } metadataLogger.Debug("group metadata") for topic, partitions := range member.Assignment { for _, partition := range partitions { helpers.TimeoutSendStorageRequest(module.App.StorageChannel, &protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOwner, Cluster: module.cluster, Topic: topic, Partition: partition, Group: group, Owner: member.ClientHost, ClientID: member.ClientID, }, 1) } } } } func decodeMetadataValueHeader(buf *bytes.Buffer) (metadataHeader, string) { var err error metadataHeader := metadataHeader{} metadataHeader.ProtocolType, err = readString(buf) if err != nil { return metadataHeader, "protocol_type" } err = binary.Read(buf, binary.BigEndian, &metadataHeader.Generation) if err != nil { return metadataHeader, "generation" } metadataHeader.Protocol, err = readString(buf) if err != nil { return metadataHeader, "protocol" } metadataHeader.Leader, err = readString(buf) if err != nil { return metadataHeader, "leader" } return metadataHeader, "" } func decodeMetadataMember(buf *bytes.Buffer, memberVersion int16) (metadataMember, string) { var err error memberMetadata := metadataMember{} memberMetadata.MemberID, err = readString(buf) if err != nil { return memberMetadata, "member_id" } memberMetadata.ClientID, err = readString(buf) if err != nil { return memberMetadata, "client_id" } memberMetadata.ClientHost, err = readString(buf) if err != nil { return memberMetadata, "client_host" } if memberVersion == 1 { err = binary.Read(buf, binary.BigEndian, &memberMetadata.RebalanceTimeout) if err != nil { return memberMetadata, "rebalance_timeout" } } err = binary.Read(buf, binary.BigEndian, &memberMetadata.SessionTimeout) if err != nil { return memberMetadata, "session_timeout" } var subscriptionBytes int32 err = binary.Read(buf, binary.BigEndian, &subscriptionBytes) if err != nil { return memberMetadata, "subscription_bytes" } if subscriptionBytes > 0 { buf.Next(int(subscriptionBytes)) } var assignmentBytes int32 err = binary.Read(buf, binary.BigEndian, &assignmentBytes) if err != nil { return memberMetadata, "assignment_bytes" } if assignmentBytes > 0 { assignmentData := buf.Next(int(assignmentBytes)) assignmentBuf := bytes.NewBuffer(assignmentData) var consumerProtocolVersion int16 err = binary.Read(assignmentBuf, binary.BigEndian, &consumerProtocolVersion) if err != nil { return memberMetadata, "consumer_protocol_version" } if consumerProtocolVersion < 0 { return memberMetadata, "consumer_protocol_version" } assignment, errorAt := decodeMemberAssignmentV0(assignmentBuf) if errorAt != "" { return memberMetadata, "assignment" } memberMetadata.Assignment = assignment } return memberMetadata, "" } func decodeMemberAssignmentV0(buf *bytes.Buffer) (map[string][]int32, string) { var err error var topics map[string][]int32 var numTopics, numPartitions, partitionID, userDataLen int32 err = binary.Read(buf, binary.BigEndian, &numTopics) if err != nil { return topics, "assignment_topic_count" } topicCount := int(numTopics) topics = make(map[string][]int32, numTopics) for i := 0; i < topicCount; i++ { topicName, err := readString(buf) if err != nil { return topics, "topic_name" } err = binary.Read(buf, binary.BigEndian, &numPartitions) if err != nil { return topics, "assignment_partition_count" } partitionCount := int(numPartitions) topics[topicName] = make([]int32, numPartitions) for j := 0; j < partitionCount; j++ { err = binary.Read(buf, binary.BigEndian, &partitionID) if err != nil { return topics, "assignment_partition_id" } topics[topicName][j] = int32(partitionID) } } err = binary.Read(buf, binary.BigEndian, &userDataLen) if err != nil { return topics, "user_bytes" } if userDataLen > 0 { buf.Next(int(userDataLen)) } return topics, "" } func decodeOffsetKeyV0(buf *bytes.Buffer) (offsetKey, string) { var err error offsetKey := offsetKey{} offsetKey.Group, err = readString(buf) if err != nil { return offsetKey, "group" } offsetKey.Topic, err = readString(buf) if err != nil { return offsetKey, "topic" } err = binary.Read(buf, binary.BigEndian, &offsetKey.Partition) if err != nil { return offsetKey, "partition" } return offsetKey, "" } func decodeOffsetValueV0(valueBuffer *bytes.Buffer) (offsetValue, string) { var err error offsetValue := offsetValue{} err = binary.Read(valueBuffer, binary.BigEndian, &offsetValue.Offset) if err != nil { return offsetValue, "offset" } _, err = readString(valueBuffer) if err != nil { return offsetValue, "metadata" } err = binary.Read(valueBuffer, binary.BigEndian, &offsetValue.Timestamp) if err != nil { return offsetValue, "timestamp" } return offsetValue, "" } func decodeOffsetValueV3(valueBuffer *bytes.Buffer) (offsetValue, string) { var err error offsetValue := offsetValue{} err = binary.Read(valueBuffer, binary.BigEndian, &offsetValue.Offset) if err != nil { return offsetValue, "offset" } var leaderEpoch int32 err = binary.Read(valueBuffer, binary.BigEndian, &leaderEpoch) if err != nil { return offsetValue, "leaderEpoch" } _, err = readString(valueBuffer) if err != nil { return offsetValue, "metadata" } err = binary.Read(valueBuffer, binary.BigEndian, &offsetValue.Timestamp) if err != nil { return offsetValue, "timestamp" } return offsetValue, "" } burrow-1.2.1/core/internal/consumer/kafka_client_test.go000066400000000000000000000632051343357346000234510ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package consumer import ( "bytes" "errors" "testing" "time" "github.com/stretchr/testify/assert" "github.com/Shopify/sarama" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) func fixtureModule() *KafkaClient { module := KafkaClient{ Log: zap.NewNop(), } module.App = &protocol.ApplicationContext{ StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("client-profile..client-id", "testid") viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.test.class-name", "kafka") viper.Set("consumer.test.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.test.cluster", "test") return &module } type errorTestSetBytes struct { KeyBytes []byte ValueBytes []byte } type errorTestSetBytesWithString struct { Bytes []byte ErrorAt string } func TestKafkaClient_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(KafkaClient)) } func TestKafkaClient_Configure(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") assert.NotNil(t, module.saramaConfig, "Expected saramaConfig to be populated") assert.Equal(t, "__consumer_offsets", module.offsetsTopic, "Default OffsetTopic value of __consumer_offsets did not get set") } func TestKafkaClient_Configure_BadCluster(t *testing.T) { module := fixtureModule() viper.Set("consumer.test.cluster", "nocluster") assert.Panics(t, func() { module.Configure("test", "consumer.test") }, "The code did not panic") } func TestKafkaClient_Configure_BadRegexp(t *testing.T) { module := fixtureModule() viper.Set("consumer.test.group-whitelist", "[") assert.Panics(t, func() { module.Configure("test", "consumer.test") }, "The code did not panic") } func TestKafkaClient_partitionConsumer(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") // Channels for testing messageChan := make(chan *sarama.ConsumerMessage) errorChan := make(chan *sarama.ConsumerError) consumer := &helpers.MockSaramaPartitionConsumer{} consumer.On("AsyncClose").Return() consumer.On("Messages").Return(func() <-chan *sarama.ConsumerMessage { return messageChan }()) consumer.On("Errors").Return(func() <-chan *sarama.ConsumerError { return errorChan }()) module.running.Add(1) go module.partitionConsumer(consumer) // Send a message over the error channel to make sure it doesn't block testError := &sarama.ConsumerError{ Topic: "testtopic", Partition: 0, Err: errors.New("test error"), } errorChan <- testError // Assure the partitionConsumer closes properly close(module.quitChannel) module.running.Wait() consumer.AssertExpectations(t) } func TestKafkaClient_startKafkaConsumer(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") // Channels for testing messageChan := make(chan *sarama.ConsumerMessage) errorChan := make(chan *sarama.ConsumerError) // Don't assert expectations on this - the way it goes down, they're called but don't show up mockPartitionConsumer := &helpers.MockSaramaPartitionConsumer{} mockPartitionConsumer.On("AsyncClose").Return() mockPartitionConsumer.On("Messages").Return(func() <-chan *sarama.ConsumerMessage { return messageChan }()) mockPartitionConsumer.On("Errors").Return(func() <-chan *sarama.ConsumerError { return errorChan }()) consumer := &helpers.MockSaramaConsumer{} consumer.On("ConsumePartition", "__consumer_offsets", int32(0), sarama.OffsetOldest).Return(mockPartitionConsumer, nil) client := &helpers.MockSaramaClient{} client.On("NewConsumerFromClient").Return(consumer, nil) client.On("Partitions", "__consumer_offsets").Return([]int32{0}, nil) err := module.startKafkaConsumer(client) assert.Nil(t, err, "Expected startKafkaConsumer to return no error") close(module.quitChannel) module.running.Wait() consumer.AssertExpectations(t) client.AssertExpectations(t) } func TestKafkaClient_startKafkaConsumer_FailCreateConsumer(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") // Set up the mock to return the leader broker for a test topic and partition testError := errors.New("test error") client := &helpers.MockSaramaClient{} client.On("NewConsumerFromClient").Return((*helpers.MockSaramaConsumer)(nil), testError) client.On("Close").Return(nil) err := module.startKafkaConsumer(client) client.AssertExpectations(t) assert.Equal(t, testError, err, "Expected startKafkaConsumer to return error") } func TestKafkaClient_startKafkaConsumer_FailGetPartitions(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") consumer := &helpers.MockSaramaConsumer{} testError := errors.New("test error") client := &helpers.MockSaramaClient{} client.On("NewConsumerFromClient").Return(consumer, nil) client.On("Partitions", "__consumer_offsets").Return([]int32{}, testError) client.On("Close").Return(nil) err := module.startKafkaConsumer(client) consumer.AssertExpectations(t) client.AssertExpectations(t) assert.Equal(t, testError, err, "Expected startKafkaConsumer to return error") } func TestKafkaClient_startKafkaConsumer_FailConsumePartition(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") testError := errors.New("test error") consumer := &helpers.MockSaramaConsumer{} consumer.On("ConsumePartition", "__consumer_offsets", int32(0), sarama.OffsetOldest).Return((*helpers.MockSaramaPartitionConsumer)(nil), testError) client := &helpers.MockSaramaClient{} client.On("NewConsumerFromClient").Return(consumer, nil) client.On("Partitions", "__consumer_offsets").Return([]int32{0}, nil) err := module.startKafkaConsumer(client) consumer.AssertExpectations(t) client.AssertExpectations(t) assert.Equal(t, testError, err, "Expected startKafkaConsumer to return error") } func TestKafkaClient_readString(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x04test")) result, err := readString(buf) assert.Equalf(t, "test", result, "Expected readString to return test, not %v", result) assert.Nil(t, err, "Expected readString to return no error") } func TestKafkaClient_readString_Underflow(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x05test")) result, err := readString(buf) assert.Equalf(t, "", result, "Expected readString to return empty string, not %v", result) assert.NotNil(t, err, "Expected readString to return an error") } func TestKafkaClient_decodeMetadataValueHeader(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader")) result, errorAt := decodeMetadataValueHeader(buf) assert.Equalf(t, "testtype", result.ProtocolType, "Expected ProtocolType to be testtype, not %v", result.ProtocolType) assert.Equalf(t, int32(1), result.Generation, "Expected Generation to be 1, not %v", result.Generation) assert.Equalf(t, "testprotocol", result.Protocol, "Expected Protocol to be testprotocol, not %v", result.Protocol) assert.Equalf(t, "testleader", result.Leader, "Expected Leader to be testleader, not %v", result.Leader) assert.Equalf(t, "", errorAt, "Expected decodeMetadataValueHeader to return empty errorAt, not %v", errorAt) } var decodeMetadataValueHeaderErrors = []errorTestSetBytesWithString{ {[]byte("\x00\x08testt"), "protocol_type"}, {[]byte("\x00\x08testtype\x00\x00"), "generation"}, {[]byte("\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestp"), "protocol"}, {[]byte("\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atest"), "leader"}, } func TestKafkaClient_decodeMetadataValueHeader_Errors(t *testing.T) { for _, values := range decodeMetadataValueHeaderErrors { _, errorAt := decodeMetadataValueHeader(bytes.NewBuffer(values.Bytes)) assert.Equalf(t, values.ErrorAt, errorAt, "Expected errorAt to be %v, not %v", values.ErrorAt, errorAt) } } func TestKafkaClient_decodeMetadataMember(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00")) result, errorAt := decodeMetadataMember(buf, 1) assert.Equalf(t, "", errorAt, "Expected decodeMetadataMember to return empty errorAt, not %v", errorAt) assert.Equalf(t, "testmemberid", result.MemberID, "Expected MemberID to be testmemberid, not %v", result.MemberID) assert.Equalf(t, "testclientid", result.ClientID, "Expected ClientID to be testclientid, not %v", result.ClientID) assert.Equalf(t, "testclienthost", result.ClientHost, "Expected ClientHost to be testclienthost, not %v", result.ClientHost) assert.Equalf(t, int32(4), result.RebalanceTimeout, "Expected RebalanceTimeout to be 4, not %v", result.RebalanceTimeout) assert.Equalf(t, int32(8), result.SessionTimeout, "Expected SessionTimeout to be 8, not %v", result.SessionTimeout) } var decodeMetadataMemberErrors = []errorTestSetBytesWithString{ {[]byte("\x00\x0ctestmemb"), "member_id"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclie"), "client_id"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestcl"), "client_host"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00"), "rebalance_timeout"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00"), "session_timeout"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00"), "subscription_bytes"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00"), "assignment_bytes"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x16\xff\xff\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00"), "consumer_protocol_version"}, {[]byte("\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00\x00\x00\x01\x00\x07topic1\x00\x00\x00\x01\x00\x00\x00\x00"), "assignment"}, } func TestKafkaClient_decodeMetadataMember_Errors(t *testing.T) { for _, values := range decodeMetadataMemberErrors { _, errorAt := decodeMetadataMember(bytes.NewBuffer(values.Bytes), 1) assert.Equalf(t, values.ErrorAt, errorAt, "Expected errorAt to be %v, not %v", values.ErrorAt, errorAt) } } func TestKafkaClient_decodeMemberAssignmentV0(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00")) assignment, errorAt := decodeMemberAssignmentV0(buf) assert.Equalf(t, "", errorAt, "Expected decodeMemberAssignmentV0 to return empty errorAt, not %v", errorAt) assert.Lenf(t, assignment, 1, "Expected Assignment to have 1 topic, not %v", len(assignment)) topic, ok := assignment["topic1"] assert.True(t, ok, "Expected to find topic1 in Assignment") assert.Lenf(t, topic, 1, "Expected topic1 to have 1 partition, not %v", len(topic)) assert.Equalf(t, int32(0), topic[0], "Expected partition ID to be 0, not %v", topic[0]) } var decodeMemberAssignmentV0Errors = []errorTestSetBytesWithString{ {[]byte("\x00\x00\x00"), "assignment_topic_count"}, {[]byte("\x00\x00\x00\x01\x00\x06top"), "topic_name"}, {[]byte("\x00\x00\x00\x01\x00\x06topic1\x00\x00"), "assignment_partition_count"}, {[]byte("\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00"), "assignment_partition_id"}, } func TestKafkaClient_decodeMemberAssignmentV0_Errors(t *testing.T) { for _, values := range decodeMemberAssignmentV0Errors { _, errorAt := decodeMemberAssignmentV0(bytes.NewBuffer(values.Bytes)) assert.Equalf(t, values.ErrorAt, errorAt, "Expected errorAt to be %v, not %v", values.ErrorAt, errorAt) } } func TestKafkaClient_decodeOffsetKeyV0(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x09testgroup\x00\x09testtopic\x00\x00\x00\x0b")) result, errorAt := decodeOffsetKeyV0(buf) assert.Equalf(t, "", errorAt, "Expected decodeOffsetKeyV0 to return empty errorAt, not %v", errorAt) assert.Equalf(t, "testgroup", result.Group, "Expected Group to be testgroup, not %v", result.Group) assert.Equalf(t, "testtopic", result.Topic, "Expected Topic to be testtopic, not %v", result.Topic) assert.Equalf(t, int32(11), result.Partition, "Expected Partition to be 11, not %v", result.Partition) } var decodeOffsetKeyV0Errors = []errorTestSetBytesWithString{ {[]byte("\x00\x09testg"), "group"}, {[]byte("\x00\x09testgroup\x00\x09testto"), "topic"}, {[]byte("\x00\x09testgroup\x00\x09testtopic\x00\x00"), "partition"}, } func TestKafkaClient_decodeOffsetKeyV0_Errors(t *testing.T) { for _, values := range decodeOffsetKeyV0Errors { _, errorAt := decodeOffsetKeyV0(bytes.NewBuffer(values.Bytes)) assert.Equalf(t, values.ErrorAt, errorAt, "Expected errorAt to be %v, not %v", values.ErrorAt, errorAt) } } func TestKafkaClient_decodeOffsetValueV0(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65")) result, errorAt := decodeOffsetValueV0(buf) assert.Equalf(t, "", errorAt, "Expected decodeOffsetValueV0 to return empty errorAt, not %v", errorAt) assert.Equalf(t, int64(8372), result.Offset, "Expected Offset to be 8372, not %v", result.Offset) assert.Equalf(t, int64(1637), result.Timestamp, "Expected Timestamp to be 1637, not %v", result.Timestamp) } var decodeOffsetValueV0Errors = []errorTestSetBytesWithString{ {[]byte("\x00\x00\x00\x00\x00"), "offset"}, {[]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08tes"), "metadata"}, {[]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00"), "timestamp"}, } func TestKafkaClient_decodeOffsetValueV0_Errors(t *testing.T) { for _, values := range decodeOffsetValueV0Errors { _, errorAt := decodeOffsetValueV0(bytes.NewBuffer(values.Bytes)) assert.Equalf(t, values.ErrorAt, errorAt, "Expected errorAt to be %v, not %v", values.ErrorAt, errorAt) } } func TestKafkaClient_decodeOffsetValueV3(t *testing.T) { buf := bytes.NewBuffer([]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x00\x00\x00\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65")) result, errorAt := decodeOffsetValueV3(buf) assert.Equalf(t, "", errorAt, "Expected decodeOffsetValueV3 to return empty errorAt, not %v", errorAt) assert.Equalf(t, int64(8372), result.Offset, "Expected Offset to be 8372, not %v", result.Offset) assert.Equalf(t, int64(1637), result.Timestamp, "Expected Timestamp to be 1637, not %v", result.Timestamp) } var decodeOffsetValueV3Errors = []errorTestSetBytesWithString{ {[]byte("\x00\x00\x00\x00\x00"), "offset"}, {[]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x00\x00"), "leaderEpoch"}, {[]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08tes"), "metadata"}, {[]byte("\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x00\x00\x00\x00\x08testdata\x00\x00\x00\x00"), "timestamp"}, } func TestKafkaClient_decodeOffsetValueV3_Errors(t *testing.T) { for _, values := range decodeOffsetValueV3Errors { _, errorAt := decodeOffsetValueV3(bytes.NewBuffer(values.Bytes)) assert.Equalf(t, values.ErrorAt, errorAt, "Expected errorAt to be %v, not %v", values.ErrorAt, errorAt) } } func TestKafkaClient_decodeKeyAndOffset(t *testing.T) { module := fixtureModule() viper.Set("consumer.test.group-whitelist", "test.*") module.Configure("test", "consumer.test") keyBuf := bytes.NewBuffer([]byte("\x00\x09testgroup\x00\x09testtopic\x00\x00\x00\x0b")) valueBytes := []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65") go module.decodeKeyAndOffset(keyBuf, valueBytes, zap.NewNop()) request := <-module.App.StorageChannel assert.Equalf(t, protocol.StorageSetConsumerOffset, request.RequestType, "Expected request sent with type StorageSetConsumerOffset, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "testtopic", request.Topic, "Expected request sent with topic testtopic, not %v", request.Topic) assert.Equalf(t, int32(11), request.Partition, "Expected request sent with partition 0, not %v", request.Partition) assert.Equalf(t, "testgroup", request.Group, "Expected request sent with Group testgroup, not %v", request.Group) assert.Equalf(t, int64(8372), request.Offset, "Expected Offset to be 8372, not %v", request.Offset) assert.Equalf(t, int64(1637), request.Timestamp, "Expected Timestamp to be 1637, not %v", request.Timestamp) } var decodeKeyAndOffsetErrors = []errorTestSetBytes{ {[]byte("\x00\x09testgroup\x00\x09testt"), []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65")}, {[]byte("\x00\x09testgroup\x00\x09testtopic\x00\x00\x00\x0b"), []byte("\x00")}, {[]byte("\x00\x09testgroup\x00\x09testtopic\x00\x00\x00\x0b"), []byte("\x00\x02\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65")}, } func TestKafkaClient_decodeKeyAndOffset_BadValueVersion(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") for _, values := range decodeKeyAndOffsetErrors { // Should not timeout module.decodeKeyAndOffset(bytes.NewBuffer(values.KeyBytes), values.ValueBytes, zap.NewNop()) } } func TestKafkaClient_decodeKeyAndOffset_Whitelist(t *testing.T) { module := fixtureModule() viper.Set("consumer.test.group-whitelist", "test.*") module.Configure("test", "consumer.test") keyBuf := bytes.NewBuffer([]byte("\x00\x0ddropthisgroup\x00\x09testtopic\x00\x00\x00\x0b")) valueBytes := []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65") // Should not timeout as the group should be dropped by the whitelist module.decodeKeyAndOffset(keyBuf, valueBytes, zap.NewNop()) } func TestKafkaClient_decodeAndSendOffset_ErrorValue(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") offsetKey := offsetKey{ Group: "testgroup", Topic: "testtopic", Partition: 11, } valueBuf := bytes.NewBuffer([]byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testd")) module.decodeAndSendOffset(offsetKey, valueBuf, zap.NewNop(), decodeOffsetValueV0) // Should not timeout } func TestKafkaClient_decodeGroupMetadata(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") keyBuf := bytes.NewBuffer([]byte("\x00\x09testgroup")) valueBytes := []byte("\x00\x01\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00\x01\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x0b\x00\x00\x00\x00") go module.decodeGroupMetadata(keyBuf, valueBytes, zap.NewNop()) request := <-module.App.StorageChannel assert.Equalf(t, protocol.StorageSetConsumerOwner, request.RequestType, "Expected request sent with type StorageSetConsumerOwner, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "topic1", request.Topic, "Expected request sent with topic testtopic, not %v", request.Topic) assert.Equalf(t, int32(11), request.Partition, "Expected request sent with partition 0, not %v", request.Partition) assert.Equalf(t, "testgroup", request.Group, "Expected request sent with Group testgroup, not %v", request.Group) assert.Equalf(t, "testclienthost", request.Owner, "Expected request sent with Owner testclienthost, not %v", request.Owner) assert.Equalf(t, "testclientid", request.ClientID, "Expected request set with ClientID testclientid, not %v", request.ClientID) } var decodeGroupMetadataErrors = []errorTestSetBytes{ {[]byte("\x00\x09testg"), []byte("\x00\x01\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00\x01\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x0b")}, {[]byte("\x00\x09testgroup"), []byte("\x00")}, {[]byte("\x00\x09testgroup"), []byte("\x00\x02\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00\x01\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x0b")}, {[]byte("\x00\x09testgroup"), []byte("\x00\x01\x00\x08test")}, {[]byte("\x00\x09testgroup"), []byte("\x00\x01\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00")}, } func TestKafkaClient_decodeGroupMetadata_Errors(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") for _, values := range decodeGroupMetadataErrors { // Should not timeout module.decodeGroupMetadata(bytes.NewBuffer(values.KeyBytes), values.ValueBytes, zap.NewNop()) } } var decodeAndSendGroupMetadataErrors = [][]byte{ []byte("\x00\x08test"), []byte("\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00"), []byte("\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00\x01\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclie"), } func TestKafkaClient_decodeAndSendGroupMetadata_Errors(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") for _, value := range decodeAndSendGroupMetadataErrors { // Should not timeout module.decodeAndSendGroupMetadata(1, "testgroup", bytes.NewBuffer(value), zap.NewNop()) } } func TestKafkaClient_processConsumerOffsetsMessage_Offset(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") msg := &sarama.ConsumerMessage{ Key: []byte("\x00\x02\x00\x09testgroup"), Value: []byte("\x00\x01\x00\x08testtype\x00\x00\x00\x01\x00\x0ctestprotocol\x00\x0atestleader\x00\x00\x00\x01\x00\x0ctestmemberid\x00\x0ctestclientid\x00\x0etestclienthost\x00\x00\x00\x04\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x01\x00\x06topic1\x00\x00\x00\x01\x00\x00\x00\x0b\x00\x00\x00\x00"), Topic: "__consumer_offsets", Partition: 0, Offset: 8232, Timestamp: time.Now(), } go module.processConsumerOffsetsMessage(msg) request := <-module.App.StorageChannel assert.Equalf(t, protocol.StorageSetConsumerOwner, request.RequestType, "Expected request sent with type StorageSetConsumerOwner, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "topic1", request.Topic, "Expected request sent with topic testtopic, not %v", request.Topic) assert.Equalf(t, int32(11), request.Partition, "Expected request sent with partition 0, not %v", request.Partition) assert.Equalf(t, "testgroup", request.Group, "Expected request sent with Group testgroup, not %v", request.Group) assert.Equalf(t, "testclienthost", request.Owner, "Expected request sent with Owner testclienthost, not %v", request.Owner) assert.Equalf(t, "testclientid", request.ClientID, "Expected request set with ClientID testclientid, no %v", request.ClientID) } func TestKafkaClient_processConsumerOffsetsMessage_Metadata(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") msg := &sarama.ConsumerMessage{ Key: []byte("\x00\x01\x00\x09testgroup\x00\x09testtopic\x00\x00\x00\x0b"), Value: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65"), Topic: "__consumer_offsets", Partition: 0, Offset: 8232, Timestamp: time.Now(), } go module.processConsumerOffsetsMessage(msg) request := <-module.App.StorageChannel assert.Equalf(t, protocol.StorageSetConsumerOffset, request.RequestType, "Expected request sent with type StorageSetConsumerOffset, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "testtopic", request.Topic, "Expected request sent with topic testtopic, not %v", request.Topic) assert.Equalf(t, int32(11), request.Partition, "Expected request sent with partition 0, not %v", request.Partition) assert.Equalf(t, "testgroup", request.Group, "Expected request sent with Group testgroup, not %v", request.Group) assert.Equalf(t, int64(8372), request.Offset, "Expected Offset to be 8372, not %v", request.Offset) assert.Equalf(t, int64(1637), request.Timestamp, "Expected Timestamp to be 1637, not %v", request.Timestamp) } var processConsumerOffsetsMessageErrors = []errorTestSetBytes{ {[]byte("\x00"), []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65")}, {[]byte("\x00\x03\x00\x09testgroup\x00\x09testtopic\x00\x00\x00\x0b"), []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x20\xb4\x00\x08testdata\x00\x00\x00\x00\x00\x00\x06\x65")}, } func TestKafkaClient_processConsumerOffsetsMessage_Errors(t *testing.T) { module := fixtureModule() module.Configure("test", "consumer.test") for _, values := range processConsumerOffsetsMessageErrors { msg := &sarama.ConsumerMessage{ Key: values.KeyBytes, Value: values.ValueBytes, Topic: "__consumer_offsets", Partition: 0, Offset: 8232, Timestamp: time.Now(), } // Should not timeout module.processConsumerOffsetsMessage(msg) } } burrow-1.2.1/core/internal/consumer/kafka_zk_client.go000066400000000000000000000363621343357346000231220ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package consumer import ( "regexp" "strconv" "sync" "time" "github.com/samuel/go-zookeeper/zk" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) type topicList struct { topics map[string]*partitionCount lock *sync.RWMutex } type partitionCount struct { count int32 lock *sync.Mutex } // KafkaZkClient is a consumer module which connects to the Zookeeper ensemble where an Apache Kafka cluster maintains // metadata, and reads consumer group information from the /consumers tree (older ZK-based consumers). It uses watches // to monitor every group and offset, and the information is forwarded to the storage subsystem for use in evaluations. type KafkaZkClient struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string cluster string servers []string zookeeperTimeout int zookeeperPath string zk protocol.ZookeeperClient areWatchesSet bool running *sync.WaitGroup groupLock *sync.RWMutex groupList map[string]*topicList groupWhitelist *regexp.Regexp groupBlacklist *regexp.Regexp connectFunc func([]string, time.Duration, *zap.Logger) (protocol.ZookeeperClient, <-chan zk.Event, error) } // Configure validates the configuration for the consumer. At minimum, there must be a cluster name to which these // consumers belong, as well as a list of servers provided for the Zookeeper ensemble, of the form host:port. If not // explicitly configured, it is assumed that the Kafka cluster metadata is present in the ensemble root path. If the // cluster name is unknown, or if the server list is missing or invalid, this func will panic. func (module *KafkaZkClient) Configure(name string, configRoot string) { module.Log.Info("configuring") module.name = name module.running = &sync.WaitGroup{} module.groupLock = &sync.RWMutex{} module.groupList = make(map[string]*topicList) module.connectFunc = helpers.ZookeeperConnect module.servers = viper.GetStringSlice(configRoot + ".servers") if len(module.servers) == 0 { panic("No Zookeeper servers specified for consumer " + module.name) } else if !helpers.ValidateHostList(module.servers) { panic("Consumer '" + name + "' has one or more improperly formatted servers (must be host:port)") } // Set defaults for configs if needed, and get them viper.SetDefault(configRoot+".zookeeper-timeout", 30) module.zookeeperTimeout = viper.GetInt(configRoot + ".zookeeper-timeout") module.zookeeperPath = viper.GetString(configRoot+".zookeeper-path") + "/consumers" module.cluster = viper.GetString(configRoot + ".cluster") if !helpers.ValidateZookeeperPath(module.zookeeperPath) { panic("Consumer '" + name + "' has a bad zookeeper path configuration") } whitelist := viper.GetString(configRoot + ".group-whitelist") if whitelist != "" { re, err := regexp.Compile(whitelist) if err != nil { module.Log.Panic("Failed to compile group whitelist") panic(err) } module.groupWhitelist = re } blacklist := viper.GetString(configRoot + ".group-blacklist") if blacklist != "" { re, err := regexp.Compile(blacklist) if err != nil { module.Log.Panic("Failed to compile group blacklist") panic(err) } module.groupBlacklist = re } } // Start connects to the Zookeeper ensemble configured. Any error connecting to the cluster is returned to the caller. // Once the client is set up, the consumer group list is enumerated and watches are set up for each group, topic, // partition, and offset. A goroutine is also started to monitor the Zookeeper connection state, and reset the watches // in the case the the session expires. func (module *KafkaZkClient) Start() error { module.Log.Info("starting") zkconn, connEventChan, err := module.connectFunc(module.servers, time.Duration(module.zookeeperTimeout)*time.Second, module.Log) if err != nil { return err } module.zk = zkconn // Set up all groups initially (we can't count on catching the first CONNECTED event module.running.Add(1) module.resetGroupListWatchAndAdd(false) module.areWatchesSet = true // Start up a func to watch for connection state changes and reset all the watches when needed module.running.Add(1) go module.connectionStateWatcher(connEventChan) return nil } // Stop closes the Zookeeper client. func (module *KafkaZkClient) Stop() error { module.Log.Info("stopping") // Closing the ZK client will invalidate all the watches, which will close all the running goroutines module.zk.Close() module.running.Wait() return nil } func (module *KafkaZkClient) connectionStateWatcher(eventChan <-chan zk.Event) { defer module.running.Done() for event := range eventChan { if event.Type == zk.EventSession { switch event.State { case zk.StateExpired: module.Log.Error("session expired") module.areWatchesSet = false case zk.StateConnected: if !module.areWatchesSet { module.Log.Info("reinitializing watches") module.groupLock.Lock() module.groupList = make(map[string]*topicList) module.groupLock.Unlock() module.running.Add(1) go module.resetGroupListWatchAndAdd(false) } } } } } func (module *KafkaZkClient) acceptConsumerGroup(group string) bool { if (module.groupWhitelist != nil) && (!module.groupWhitelist.MatchString(group)) { return false } if (module.groupBlacklist != nil) && module.groupBlacklist.MatchString(group) { return false } return true } // This is a simple goroutine that will wait for an event on a watch channnel and then exit. It's here so that when // we set a watch that we don't care about (from an ExistsW on a node that already exists), we can drain it properly. func drainEventChannel(eventChan <-chan zk.Event) { <-eventChan } func (module *KafkaZkClient) waitForNodeToExist(zkPath string, Logger *zap.Logger) bool { nodeExists, _, existsWatchChan, err := module.zk.ExistsW(zkPath) if err != nil { // This is a real error (since NoNode will not return an error) Logger.Debug("failed to check existence of znode", zap.String("path", zkPath), zap.String("error", err.Error()), ) return false } if nodeExists { // The node already exists, just drain the data watch that got created whenever it fires go drainEventChannel(existsWatchChan) return true } // Wait for the node to exist Logger.Debug("waiting for node to exist", zap.String("path", zkPath)) event := <-existsWatchChan if event.Type == zk.EventNotWatching { // Watch is gone, so we're gone too Logger.Debug("exists watch invalidated", zap.String("path", zkPath), ) return false } return true } func (module *KafkaZkClient) watchGroupList(eventChan <-chan zk.Event) { defer module.running.Done() event := <-eventChan if event.Type == zk.EventNotWatching { // We're done here module.Log.Debug("group list watch invalidated") return } module.Log.Debug("group list watch fired", zap.Int("event_type", int(event.Type))) module.running.Add(1) go module.resetGroupListWatchAndAdd(event.Type != zk.EventNodeChildrenChanged) } func (module *KafkaZkClient) resetGroupListWatchAndAdd(resetOnly bool) { defer module.running.Done() // Get the current group list and reset our watch consumerGroups, _, groupListEventChan, err := module.zk.ChildrenW(module.zookeeperPath) if err != nil { // Can't read the consumers path. Bail for now module.Log.Error("failed to list groups", zap.String("error", err.Error())) return } module.running.Add(1) go module.watchGroupList(groupListEventChan) if !resetOnly { // Check for any new groups and create the watches for them module.groupLock.Lock() defer module.groupLock.Unlock() for _, group := range consumerGroups { if !module.acceptConsumerGroup(group) { module.Log.Debug("skip group", zap.String("group", group), zap.String("reason", "whitelist"), ) continue } if module.groupList[group] == nil { module.groupList[group] = &topicList{ topics: make(map[string]*partitionCount), lock: &sync.RWMutex{}, } module.Log.Debug("add group", zap.String("group", group), ) module.running.Add(1) go module.resetTopicListWatchAndAdd(group, false) } } } } func (module *KafkaZkClient) watchTopicList(group string, eventChan <-chan zk.Event) { defer module.running.Done() event := <-eventChan if event.Type == zk.EventNotWatching { // We're done here module.Log.Debug("topic list watch invalidated", zap.String("group", group)) return } module.Log.Debug("topic list watch fired", zap.String("group", group), zap.Int("event_type", int(event.Type)), ) module.running.Add(1) go module.resetTopicListWatchAndAdd(group, event.Type != zk.EventNodeChildrenChanged) } func (module *KafkaZkClient) resetTopicListWatchAndAdd(group string, resetOnly bool) { defer module.running.Done() // Wait for the offsets znode for this group to exist. We need to do this because the previous child watch // fires on /consumers/(group) existing, but here we try to read /consumers/(group)/offsets (which might not exist // yet) zkPath := module.zookeeperPath + "/" + group + "/offsets" Logger := module.Log.With(zap.String("group", group)) if !module.waitForNodeToExist(zkPath, Logger) { // There was an error checking node existence, so we can't continue return } // Get the current group topic list and reset our watch groupTopics, _, topicListEventChan, err := module.zk.ChildrenW(zkPath) if err != nil { Logger.Debug("failed to read topic list", zap.String("error", err.Error())) return } module.running.Add(1) go module.watchTopicList(group, topicListEventChan) if !resetOnly { // Check for any new topics and create the watches for them module.groupLock.RLock() defer module.groupLock.RUnlock() module.groupList[group].lock.Lock() defer module.groupList[group].lock.Unlock() for _, topic := range groupTopics { if module.groupList[group].topics[topic] == nil { module.groupList[group].topics[topic] = &partitionCount{ count: 0, lock: &sync.Mutex{}, } Logger.Debug("add topic", zap.String("topic", topic)) module.running.Add(1) go module.resetPartitionListWatchAndAdd(group, topic, false) } } } } func (module *KafkaZkClient) watchPartitionList(group string, topic string, eventChan <-chan zk.Event) { defer module.running.Done() event := <-eventChan if event.Type == zk.EventNotWatching { // We're done here module.Log.Debug("partition list watch invalidated", zap.String("group", group), zap.String("topic", topic), ) return } module.Log.Debug("partition list watch fired", zap.String("group", group), zap.String("topic", topic), zap.Int("event_type", int(event.Type)), ) module.running.Add(1) go module.resetPartitionListWatchAndAdd(group, topic, event.Type != zk.EventNodeChildrenChanged) } func (module *KafkaZkClient) resetPartitionListWatchAndAdd(group string, topic string, resetOnly bool) { defer module.running.Done() // Get the current topic partition list and reset our watch topicPartitions, _, partitionListEventChan, err := module.zk.ChildrenW(module.zookeeperPath + "/" + group + "/offsets/" + topic) if err != nil { // Can't read the partition list path. Bail for now module.Log.Warn("failed to read partitions", zap.String("group", group), zap.String("topic", topic), zap.String("error", err.Error()), ) return } module.running.Add(1) go module.watchPartitionList(group, topic, partitionListEventChan) if !resetOnly { // Check for any new partitions and create the watches for them module.groupLock.RLock() defer module.groupLock.RUnlock() module.groupList[group].lock.RLock() defer module.groupList[group].lock.RUnlock() module.groupList[group].topics[topic].lock.Lock() defer module.groupList[group].topics[topic].lock.Unlock() if int32(len(topicPartitions)) >= module.groupList[group].topics[topic].count { for i := module.groupList[group].topics[topic].count; i < int32(len(topicPartitions)); i++ { module.Log.Debug("add partition", zap.String("group", group), zap.String("topic", topic), zap.Int32("partition", i), ) module.running.Add(1) module.resetOffsetWatchAndSend(group, topic, i, false) } module.groupList[group].topics[topic].count = int32(len(topicPartitions)) } } } func (module *KafkaZkClient) watchOffset(group string, topic string, partition int32, eventChan <-chan zk.Event) { defer module.running.Done() event := <-eventChan if event.Type == zk.EventNotWatching { // We're done here module.Log.Debug("offset watch invalidated", zap.String("group", group), zap.String("topic", topic), zap.Int32("partition", partition), ) return } module.Log.Debug("offset watch fired", zap.String("group", group), zap.String("topic", topic), zap.Int32("partition", partition), zap.Int("event_type", int(event.Type)), ) module.running.Add(1) go module.resetOffsetWatchAndSend(group, topic, partition, event.Type != zk.EventNodeDataChanged) } func (module *KafkaZkClient) resetOffsetWatchAndSend(group string, topic string, partition int32, resetOnly bool) { defer module.running.Done() // Get the current offset and reset our watch offsetString, offsetStat, offsetEventChan, err := module.zk.GetW(module.zookeeperPath + "/" + group + "/offsets/" + topic + "/" + strconv.FormatInt(int64(partition), 10)) if err != nil { // Can't read the partition offset path. Bail for now module.Log.Warn("failed to read offset", zap.String("group", group), zap.String("topic", topic), zap.Int32("partition", partition), zap.String("error", err.Error()), ) return } module.running.Add(1) go module.watchOffset(group, topic, partition, offsetEventChan) if !resetOnly { offset, err := strconv.ParseInt(string(offsetString), 10, 64) if err != nil { // Badly formatted offset module.Log.Error("badly formatted offset", zap.String("group", group), zap.String("topic", topic), zap.Int32("partition", partition), zap.ByteString("offset_string", offsetString), zap.String("error", err.Error()), ) return } // Send the offset to the storage module partitionOffset := &protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: module.cluster, Topic: topic, Partition: int32(partition), Group: group, Timestamp: offsetStat.Mtime, Offset: offset, } module.Log.Debug("consumer offset", zap.String("group", group), zap.String("topic", topic), zap.Int32("partition", partition), zap.Int64("offset", offset), zap.Int64("timestamp", offsetStat.Mtime), ) helpers.TimeoutSendStorageRequest(module.App.StorageChannel, partitionOffset, 1) } } burrow-1.2.1/core/internal/consumer/kafka_zk_test.go000066400000000000000000000247451343357346000226250ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package consumer import ( "errors" "time" "github.com/stretchr/testify/assert" "testing" "github.com/samuel/go-zookeeper/zk" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" "github.com/stretchr/testify/mock" "sync" ) func fixtureKafkaZkModule() *KafkaZkClient { module := KafkaZkClient{ Log: zap.NewNop(), } module.App = &protocol.ApplicationContext{ StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.test.class-name", "kafka_zk") viper.Set("consumer.test.servers", []string{"broker1.example.com:1234"}) viper.Set("consumer.test.cluster", "test") return &module } func TestKafkaZkClient_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(KafkaZkClient)) } func TestKafkaZkClient_Configure(t *testing.T) { module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") assert.Equal(t, "/consumers", module.zookeeperPath, "Expected ZookeeperPath to get set to '/consumers', not %v", module.zookeeperPath) assert.Equal(t, int(30), module.zookeeperTimeout, "Default ZookeeperTimeout value of 30 did not get set") } func TestKafkaZkClient_Configure_BadRegexp(t *testing.T) { module := fixtureModule() viper.Set("consumer.test.group-whitelist", "[") assert.Panics(t, func() { module.Configure("test", "consumer.test") }, "The code did not panic") } func TestKafkaZkClient_Start(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{ EventChannel: make(chan zk.Event), } module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") module.connectFunc = mockZookeeper.MockZookeeperConnect watchEventChan := make(chan zk.Event) mockZookeeper.On("ChildrenW", module.zookeeperPath).Return([]string{}, &zk.Stat{}, func() <-chan zk.Event { return watchEventChan }(), nil) mockZookeeper.On("Close").Return().Run(func(args mock.Arguments) { watchEventChan <- zk.Event{Type: zk.EventNotWatching} close(watchEventChan) }) err := module.Start() // Check that there is something reading connection state events - this should not block mockZookeeper.EventChannel <- zk.Event{} module.Stop() assert.Nil(t, err, "Expected Start to return no error") assert.Equal(t, module.servers, mockZookeeper.Servers, "Expected ZookeeperConnect to be called with server list") assert.Equal(t, time.Duration(module.zookeeperTimeout)*time.Second, mockZookeeper.SessionTimeout, "Expected ZookeeperConnect to be called with session timeout") mockZookeeper.AssertExpectations(t) } // This tests all the watchers - each one will be called in turn and set, and we assure that they're all closing properly func TestKafkaZkClient_watchGroupList(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} module := fixtureKafkaZkModule() viper.Set("consumer.test.group-whitelist", "test.*") module.Configure("test", "consumer.test") module.zk = &mockZookeeper offsetStat := &zk.Stat{Mtime: 894859} newGroupChan := make(chan zk.Event) topicExistsChan := make(chan zk.Event) newTopicChan := make(chan zk.Event) newPartitionChan := make(chan zk.Event) newOffsetChan := make(chan zk.Event) mockZookeeper.On("ChildrenW", "/consumers").Return([]string{"testgroup"}, offsetStat, func() <-chan zk.Event { return newGroupChan }(), nil) mockZookeeper.On("ChildrenW", "/consumers/testgroup/offsets").Return([]string{"testtopic"}, offsetStat, func() <-chan zk.Event { return newTopicChan }(), nil) mockZookeeper.On("ExistsW", "/consumers/testgroup/offsets").Return(true, offsetStat, func() <-chan zk.Event { return topicExistsChan }(), nil) mockZookeeper.On("ChildrenW", "/consumers/testgroup/offsets/testtopic").Return([]string{"0"}, offsetStat, func() <-chan zk.Event { return newPartitionChan }(), nil) mockZookeeper.On("GetW", "/consumers/testgroup/offsets/testtopic/0").Return([]byte("81234"), offsetStat, func() <-chan zk.Event { return newOffsetChan }(), nil) watchEventChan := make(chan zk.Event) wg := sync.WaitGroup{} wg.Add(1) go func() { watchEventChan <- zk.Event{ Type: zk.EventNodeChildrenChanged, State: zk.StateConnected, Path: "/consumers", } request := <-module.App.StorageChannel assert.Equalf(t, protocol.StorageSetConsumerOffset, request.RequestType, "Expected request sent with type StorageSetConsumerOffset, not %v", request.RequestType) assert.Equalf(t, "test", request.Cluster, "Expected request sent with cluster test, not %v", request.Cluster) assert.Equalf(t, "testtopic", request.Topic, "Expected request sent with topic testtopic, not %v", request.Topic) assert.Equalf(t, int32(0), request.Partition, "Expected request sent with partition 0, not %v", request.Partition) assert.Equalf(t, "testgroup", request.Group, "Expected request sent with Group testgroup, not %v", request.Group) assert.Equalf(t, int64(81234), request.Offset, "Expected Offset to be 8372, not %v", request.Offset) assert.Equalf(t, int64(894859), request.Timestamp, "Expected Timestamp to be 1637, not %v", request.Timestamp) newGroupChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/shouldntgetcalled", } newTopicChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/testgroup/offsets/shouldntgetcalled", } topicExistsChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/testgroup/offsets", } newPartitionChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/testgroup/offsets/testtopic/shouldntgetcalled", } newOffsetChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/testgroup/offsets/testtopic/1/shouldntgetcalled", } }() module.running.Add(1) module.watchGroupList(watchEventChan) module.running.Wait() mockZookeeper.AssertExpectations(t) assert.Equalf(t, int32(1), module.groupList["testgroup"].topics["testtopic"].count, "Expected partition count to be 1, not %v", module.groupList["testgroup"].topics["testtopic"].count) } func TestKafkaZkClient_resetOffsetWatchAndSend_BadPath(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} mockZookeeper.On("GetW", "/consumers/testgroup/offsets/testtopic/0").Return([]byte("81234"), (*zk.Stat)(nil), (<-chan zk.Event)(nil), errors.New("badpath")) module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") module.zk = &mockZookeeper module.running.Add(1) module.resetOffsetWatchAndSend("testgroup", "testtopic", 0, false) mockZookeeper.AssertExpectations(t) } func TestKafkaZkClient_resetOffsetWatchAndSend_BadOffset(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") module.zk = &mockZookeeper offsetStat := &zk.Stat{Mtime: 894859} newWatchEventChan := make(chan zk.Event) mockZookeeper.On("GetW", "/consumers/testgroup/offsets/testtopic/0").Return([]byte("notanumber"), offsetStat, func() <-chan zk.Event { return newWatchEventChan }(), nil) // This will block if a storage request is sent, as nothing is watching that channel module.running.Add(1) module.resetOffsetWatchAndSend("testgroup", "testtopic", 0, false) // This should not block because the watcher would have been started newWatchEventChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/testgroup/offsets/testtopic/1", } mockZookeeper.AssertExpectations(t) } func TestKafkaZkClient_resetPartitionListWatchAndAdd_BadPath(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} mockZookeeper.On("ChildrenW", "/consumers/testgroup/offsets/testtopic").Return([]string{}, (*zk.Stat)(nil), (<-chan zk.Event)(nil), errors.New("badpath")) module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") module.zk = &mockZookeeper module.running.Add(1) module.resetPartitionListWatchAndAdd("testgroup", "testtopic", false) mockZookeeper.AssertExpectations(t) } func TestKafkaZkClient_resetTopicListWatchAndAdd_BadPath(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} topicExistsChan := make(chan zk.Event) mockZookeeper.On("ExistsW", "/consumers/testgroup/offsets").Return(false, (*zk.Stat)(nil), func() <-chan zk.Event { return topicExistsChan }(), nil) module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") module.zk = &mockZookeeper go func() { topicExistsChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/testgroup/offsets", } }() module.running.Add(1) module.resetTopicListWatchAndAdd("testgroup", false) mockZookeeper.AssertExpectations(t) } func TestKafkaZkClient_resetGroupListWatchAndAdd_BadPath(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} mockZookeeper.On("ChildrenW", "/consumers").Return([]string{}, (*zk.Stat)(nil), (<-chan zk.Event)(nil), errors.New("badpath")) module := fixtureKafkaZkModule() module.Configure("test", "consumer.test") module.zk = &mockZookeeper module.running.Add(1) module.resetGroupListWatchAndAdd(false) mockZookeeper.AssertExpectations(t) } func TestKafkaZkClient_resetGroupListWatchAndAdd_WhiteList(t *testing.T) { mockZookeeper := helpers.MockZookeeperClient{} module := fixtureKafkaZkModule() viper.Set("consumer.test.group-whitelist", "test.*") module.Configure("test", "consumer.test") module.zk = &mockZookeeper offsetStat := &zk.Stat{Mtime: 894859} newGroupChan := make(chan zk.Event) mockZookeeper.On("ChildrenW", "/consumers").Return([]string{"dropthisgroup"}, offsetStat, func() <-chan zk.Event { return newGroupChan }(), nil) module.running.Add(1) module.resetGroupListWatchAndAdd(false) newGroupChan <- zk.Event{ Type: zk.EventNotWatching, State: zk.StateConnected, Path: "/consumers/shouldntgetcalled", } mockZookeeper.AssertExpectations(t) _, ok := module.groupList["dropthisgroup"] assert.False(t, ok, "Expected group to be dropped due to whitelist") } burrow-1.2.1/core/internal/doc.go000066400000000000000000000006461343357346000167110ustar00rootroot00000000000000// Package internal - Here be dragons. // The internal package contains the bulk of Burrow's logic, including all of the coordinators and modules that have // been defined. This documentation is targeted at developers of Burrow modules to get more information about the // internal structure and how the modules fit together. It is not designed for end users, and for the most part will not // be useful. package internal burrow-1.2.1/core/internal/evaluator/000077500000000000000000000000001343357346000176115ustar00rootroot00000000000000burrow-1.2.1/core/internal/evaluator/caching.go000066400000000000000000000330131343357346000215340ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package evaluator import ( "strings" "sync" "time" "github.com/karrick/goswarm" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) // CachingEvaluator is an evaluator module that responds to evaluation requests and checks consumer status using the // standard Burrow definitions for stall, stop, and lag. The results are stored in an in-memory cache for a configurable // amount of time, in order to avoid duplication of work when multiple modules evaluate the same consumer group. type CachingEvaluator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string expireCache int minimumComplete float32 RequestChannel chan *protocol.EvaluatorRequest running sync.WaitGroup cache *goswarm.Simple } type cacheError struct { StatusCode int Reason string } func (e *cacheError) Error() string { return e.Reason } // Configure validates the configuration for the module, creates a channel to receive requests on, and sets up the // cache. If no expiration time for cache entries is set, a default value of 10 seconds is used. If there is any problem // starting the goswarm cache, this func panics. func (module *CachingEvaluator) Configure(name string, configRoot string) { module.Log.Info("configuring") module.name = name module.RequestChannel = make(chan *protocol.EvaluatorRequest) module.running = sync.WaitGroup{} // Set defaults for configs if needed viper.SetDefault(configRoot+".expire-cache", 10) module.expireCache = viper.GetInt(configRoot + ".expire-cache") module.minimumComplete = float32(viper.GetFloat64(configRoot + ".minimum-complete")) cacheExpire := time.Duration(module.expireCache) * time.Second newCache, err := goswarm.NewSimple(&goswarm.Config{ GoodExpiryDuration: cacheExpire, BadExpiryDuration: cacheExpire, Lookup: module.evaluateConsumerStatus, }) if err != nil { module.Log.Panic("Failed to start cache") panic(err) } module.cache = newCache } // GetCommunicationChannel returns the RequestChannel that has been setup for this module. func (module *CachingEvaluator) GetCommunicationChannel() chan *protocol.EvaluatorRequest { return module.RequestChannel } // Start instantiates the main loop that listens for evaluation requests and returns the result func (module *CachingEvaluator) Start() error { module.Log.Info("starting") module.running.Add(1) go module.mainLoop() return nil } // Stop closes the module's RequestChannel, which also terminates the main loop that responds to requests func (module *CachingEvaluator) Stop() error { module.Log.Info("stopping") close(module.RequestChannel) module.running.Wait() return nil } func (module *CachingEvaluator) mainLoop() { defer module.running.Done() for request := range module.RequestChannel { if request != nil { go module.getConsumerStatus(request) } } } func (module *CachingEvaluator) getConsumerStatus(request *protocol.EvaluatorRequest) { // Easier to set up the structured logger once for the request requestLogger := module.Log.With( zap.String("cluster", request.Cluster), zap.String("consumer", request.Group), zap.Bool("showall", request.ShowAll), ) result, err := module.cache.Query(request.Cluster + " " + request.Group) if err != nil { requestLogger.Info(err.Error()) // We're just returning all errors as a 404 here request.Reply <- &protocol.ConsumerGroupStatus{ Cluster: request.Cluster, Group: request.Group, Status: protocol.StatusNotFound, Complete: 1.0, Partitions: make([]*protocol.PartitionStatus, 0), Maxlag: nil, TotalLag: 0, } } else { status := result.(*protocol.ConsumerGroupStatus) if !request.ShowAll { // The requestor only wants partitions that are not StatusOK, so we need to filter the result before // returning it. However, we can't modify the original, so we need to make a new copy cachedStatus := status status = &protocol.ConsumerGroupStatus{ Cluster: cachedStatus.Cluster, Group: cachedStatus.Group, Status: cachedStatus.Status, Complete: cachedStatus.Complete, Maxlag: cachedStatus.Maxlag, TotalLag: cachedStatus.TotalLag, TotalPartitions: cachedStatus.TotalPartitions, Partitions: make([]*protocol.PartitionStatus, cachedStatus.TotalPartitions), } // Copy over any partitions that do not have the status StatusOK count := 0 for _, partition := range cachedStatus.Partitions { if partition.Status > protocol.StatusOK { status.Partitions[count] = partition count++ } } status.Partitions = status.Partitions[0:count] } requestLogger.Debug("ok") request.Reply <- status } } func (module *CachingEvaluator) evaluateConsumerStatus(clusterAndConsumer string) (interface{}, error) { // First off, we need to separate the cluster and consumer values from the string provided parts := strings.Split(clusterAndConsumer, " ") if len(parts) != 2 { module.Log.Error("query with bad clusterAndConsumer", zap.String("arg", clusterAndConsumer)) return nil, &cacheError{StatusCode: 500, Reason: "bad request"} } cluster := parts[0] consumer := parts[1] // Fetch all the consumer offset and lag information from storage storageRequest := &protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumer, Cluster: cluster, Group: consumer, Reply: make(chan interface{}), } module.App.StorageChannel <- storageRequest response := <-storageRequest.Reply if response == nil { // Either the cluster or the consumer doesn't exist. In either case, return an error module.Log.Debug("evaluation result", zap.String("cluster", cluster), zap.String("consumer", consumer), zap.String("status", protocol.StatusNotFound.String()), ) return nil, &cacheError{StatusCode: 404, Reason: "cluster or consumer not found"} } // From here out, we're going to return a non-error response, so prepare a status struct status := &protocol.ConsumerGroupStatus{ Cluster: cluster, Group: consumer, Status: protocol.StatusOK, Complete: 1.0, Maxlag: nil, TotalLag: 0, TotalPartitions: 0, } // Count up the number of partitions for this consumer first, so we can size our slice correctly topics := response.(protocol.ConsumerTopics) for _, partitions := range topics { for _, partition := range partitions { status.TotalPartitions++ status.TotalLag += partition.CurrentLag } } status.Partitions = make([]*protocol.PartitionStatus, status.TotalPartitions) count := 0 completePartitions := 0 for topic, partitions := range topics { for partitionID, partition := range partitions { partitionStatus := evaluatePartitionStatus(partition, module.minimumComplete) partitionStatus.Topic = topic partitionStatus.Partition = int32(partitionID) partitionStatus.Owner = partition.Owner if partitionStatus.Status > status.Status { // If the partition status is greater than StatusError, we just mark it as StatusError if partitionStatus.Status > protocol.StatusError { status.Status = protocol.StatusError } else { status.Status = partitionStatus.Status } } if (status.Maxlag == nil) || (partitionStatus.CurrentLag > status.Maxlag.CurrentLag) { status.Maxlag = partitionStatus } if partitionStatus.Complete == 1.0 { completePartitions++ } status.Partitions[count] = partitionStatus count++ } } // Calculate completeness as a percentage of the number of partitions that are complete if status.TotalPartitions > 0 { status.Complete = float32(completePartitions) / float32(status.TotalPartitions) } else { status.Complete = 0 } module.Log.Debug("evaluation result", zap.String("cluster", cluster), zap.String("consumer", consumer), zap.String("status", status.Status.String()), zap.Float32("complete", status.Complete), zap.Uint64("total_lag", status.TotalLag), zap.Int("total_partitions", status.TotalPartitions), ) return status, nil } func evaluatePartitionStatus(partition *protocol.ConsumerPartition, minimumComplete float32) *protocol.PartitionStatus { status := &protocol.PartitionStatus{ Status: protocol.StatusOK, CurrentLag: partition.CurrentLag, } // If there are no offsets, we can't do anything if len(partition.Offsets) == 0 { return status } // Slice the offsets to remove all nil entries (they'll be at the start) firstOffset := len(partition.Offsets) - 1 for i, offset := range partition.Offsets { if offset != nil { firstOffset = i break } } offsets := partition.Offsets[firstOffset:] // Check if we had any nil offsets, and mark the partition as incomplete if len(offsets) < len(partition.Offsets) { status.Complete = float32(len(offsets)) / float32(len(partition.Offsets)) } else { status.Complete = 1.0 } // If there are no offsets left, just return an OK result as is - we can't determine anything more if len(offsets) == 0 { return status } status.Start = offsets[0] status.End = offsets[len(offsets)-1] // If the partition does not meet the completeness threshold, just return it as OK if status.Complete >= minimumComplete { status.Status = calculatePartitionStatus(offsets, partition.BrokerOffsets, partition.CurrentLag, time.Now().Unix()) } return status } func calculatePartitionStatus(offsets []*protocol.ConsumerOffset, brokerOffsets []int64, currentLag uint64, timeNow int64) protocol.StatusConstant { // If the current lag is zero, the partition is never in error if currentLag > 0 { // Check if the partition is stopped first, as this is a problem even if the consumer had zero lag at some // point in its commit history (as the commit history could be very old). However, if the recent broker offsets // for this partition show that the consumer had zero lag recently ("intervals * offset-refresh" should be on // the order of minutes), don't consider it stopped yet. if checkIfOffsetsStopped(offsets, timeNow) && (!checkIfRecentLagZero(offsets, brokerOffsets)) { return protocol.StatusStop } // Now check if the lag was zero at any point, and skip the rest of the checks if this is true if isLagAlwaysNotZero(offsets) { // Check for errors, in order of severity starting with the worst. If any check comes back true, skip the rest if checkIfOffsetsRewind(offsets) { return protocol.StatusRewind } if checkIfOffsetsStalled(offsets) { return protocol.StatusStall } if checkIfLagNotDecreasing(offsets) { return protocol.StatusWarning } } } return protocol.StatusOK } // Rule 1 - If over the stored period, the lag is ever zero for the partition, the period is OK func isLagAlwaysNotZero(offsets []*protocol.ConsumerOffset) bool { for _, offset := range offsets { if offset.Lag == 0 { return false } } return true } // Rule 2 - If the consumer offset decreases from one interval to the next the partition is marked as a rewind (error) func checkIfOffsetsRewind(offsets []*protocol.ConsumerOffset) bool { for i := 1; i < len(offsets); i++ { if offsets[i].Offset < offsets[i-1].Offset { return true } } return false } // Rule 3 - If the difference between now and the last offset timestamp is greater than the difference between the last // and first offset timestamps, the consumer has stopped committing offsets for that partition (error) func checkIfOffsetsStopped(offsets []*protocol.ConsumerOffset, timeNow int64) bool { firstTimestamp := offsets[0].Timestamp lastTimestamp := offsets[len(offsets)-1].Timestamp return ((timeNow * 1000) - lastTimestamp) > (lastTimestamp - firstTimestamp) } // Rule 4 - If the consumer is committing offsets that do not change, it's an error (partition is stalled) // NOTE - we already checked for zero lag in Rule 1, so we know that there is currently lag for this partition func checkIfOffsetsStalled(offsets []*protocol.ConsumerOffset) bool { for i := 1; i < len(offsets); i++ { if offsets[i].Offset != offsets[i-1].Offset { return false } } return true } // Rule 5 - If the consumer offsets are advancing, but the lag is not decreasing somewhere, it's a warning (consumer is slow) func checkIfLagNotDecreasing(offsets []*protocol.ConsumerOffset) bool { for i := 1; i < len(offsets); i++ { if offsets[i].Lag < offsets[i-1].Lag { return false } } return true } // Using the most recent committed offset, return true if there was zero lag at some point in the stored broker // LEO offsets. This has the effect of returning true if the consumer was up to date on this partition in recent // (minutes) history, so it can be used to delay alerting for a short period of time. func checkIfRecentLagZero(offsets []*protocol.ConsumerOffset, brokerOffsets []int64) bool { lastOffset := offsets[len(offsets)-1].Offset for i := 0; i < len(brokerOffsets); i++ { if brokerOffsets[i] <= lastOffset { return true } } return false } burrow-1.2.1/core/internal/evaluator/caching_test.go000066400000000000000000000463461343357346000226100ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package evaluator import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/storage" "github.com/linkedin/Burrow/core/protocol" ) func fixtureModule() (*storage.Coordinator, *CachingEvaluator) { storageCoordinator := storage.CoordinatorWithOffsets() module := &CachingEvaluator{ Log: zap.NewNop(), } module.App = storageCoordinator.App module.App.EvaluatorChannel = make(chan *protocol.EvaluatorRequest) viper.Reset() viper.Set("evaluator.test.class-name", "caching") viper.Set("evaluator.test.expire-cache", 30) // Return the module without starting it, so we can test configure, start, and stop return storageCoordinator, module } func startWithTestCluster() (*storage.Coordinator, *CachingEvaluator) { storageCoordinator, module := fixtureModule() module.Configure("test", "evaluator.test") module.Start() return storageCoordinator, module } func stopTestCluster(storageCoordinator *storage.Coordinator, module *CachingEvaluator) { module.Stop() storageCoordinator.Stop() } func TestCachingEvaluator_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(CachingEvaluator)) } func TestCachingEvaluator_ImplementsEvaluatorModule(t *testing.T) { assert.Implements(t, (*Module)(nil), new(CachingEvaluator)) } func TestCachingEvaluator_Configure(t *testing.T) { storageCoordinator, module := fixtureModule() module.Configure("test", "evaluator.test") storageCoordinator.Stop() } func TestCachingEvaluator_Configure_DefaultExpireCache(t *testing.T) { storageCoordinator, module := fixtureModule() viper.Reset() viper.Set("evaluator.test.class-name", "caching") module.Configure("test", "evaluator.test") assert.Equal(t, int(10), module.expireCache, "Default ExpireCache value of 10 did not get set") storageCoordinator.Stop() } // Also tests Stop func TestCachingEvaluator_Start(t *testing.T) { storageCoordinator, module := startWithTestCluster() // We should send a request for a non-existent group, which will return a StatusNotFound request := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "nosuchgroup", ShowAll: false, } module.GetCommunicationChannel() <- request response := <-request.Reply assert.Equalf(t, response.Status, protocol.StatusNotFound, "Expected status to be NOTFOUND, not %v", response.Status.String()) stopTestCluster(storageCoordinator, module) } func TestCachingEvaluator_SingleRequest_NoShowAll(t *testing.T) { storageCoordinator, module := startWithTestCluster() request := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "testgroup", ShowAll: false, } module.GetCommunicationChannel() <- request response := <-request.Reply assert.Equalf(t, protocol.StatusOK, response.Status, "Expected status to be OK, not %v", response.Status.String()) assert.Equalf(t, float32(1.0), response.Complete, "Expected complete to be 1.0, not %v", response.Complete) assert.Equalf(t, 1, response.TotalPartitions, "Expected total_partitions to be 1, not %v", response.TotalPartitions) assert.Equalf(t, uint64(2421), response.TotalLag, "Expected total_lag to be 2421, not %v", response.TotalLag) assert.Equalf(t, "testcluster", response.Cluster, "Expected cluster to be testcluster, not %v", response.Cluster) assert.Equalf(t, "testgroup", response.Group, "Expected group to be testgroup, not %v", response.Group) assert.Lenf(t, response.Partitions, 0, "Expected 0 partition status objects, not %v", len(response.Partitions)) stopTestCluster(storageCoordinator, module) } func TestCachingEvaluator_SingleRequest_ShowAll(t *testing.T) { storageCoordinator, module := startWithTestCluster() request := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "testgroup", ShowAll: true, } module.GetCommunicationChannel() <- request response := <-request.Reply assert.Equalf(t, protocol.StatusOK, response.Status, "Expected status to be OK, not %v", response.Status.String()) assert.Equalf(t, float32(1.0), response.Complete, "Expected complete to be 1.0, not %v", response.Complete) assert.Equalf(t, 1, response.TotalPartitions, "Expected total_partitions to be 1, not %v", response.TotalPartitions) assert.Equalf(t, uint64(2421), response.TotalLag, "Expected total_lag to be 2421, not %v", response.TotalLag) assert.Equalf(t, "testcluster", response.Cluster, "Expected cluster to be testcluster, not %v", response.Cluster) assert.Equalf(t, "testgroup", response.Group, "Expected group to be testgroup, not %v", response.Group) assert.Lenf(t, response.Partitions, 1, "Expected 1 partition status objects, not %v", len(response.Partitions)) stopTestCluster(storageCoordinator, module) } func TestCachingEvaluator_SingleRequest_Incomplete(t *testing.T) { storageCoordinator, module := startWithTestCluster() request := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "testgroup2", } module.GetCommunicationChannel() <- request response := <-request.Reply assert.Equalf(t, protocol.StatusError, response.Status, "Expected status to be ERR, not %v", response.Status.String()) assert.Equalf(t, float32(0.0), response.Complete, "Expected complete to be 0.0, not %v", response.Complete) assert.Equalf(t, 1, response.TotalPartitions, "Expected total_partitions to be 1, not %v", response.TotalPartitions) assert.Equalf(t, uint64(2921), response.TotalLag, "Expected total_lag to be 2921, not %v", response.TotalLag) assert.Equalf(t, "testcluster", response.Cluster, "Expected cluster to be testcluster, not %v", response.Cluster) assert.Equalf(t, "testgroup2", response.Group, "Expected group to be testgroup2, not %v", response.Group) assert.Lenf(t, response.Partitions, 1, "Expected 1 partition status objects, not %v", len(response.Partitions)) assert.Equalf(t, float32(0.5), response.Partitions[0].Complete, "Expected partition Complete to be 0.5, not %v", response.Partitions[0].Complete) assert.Equalf(t, protocol.StatusStop, response.Partitions[0].Status, "Expected partition status to be STOP, not %v", response.Partitions[0].Status.String()) assert.NotNil(t, response.Maxlag, "Expected Maxlag to be not nil") assert.NotNil(t, response.Maxlag.Start, "Expected Maxlag.Start to be not nil") assert.Equalf(t, int64(1000), response.Maxlag.Start.Offset, "Expected Maxlag.Start.Offset to be 100, not %v", response.Maxlag.Start.Offset) assert.NotNil(t, response.Maxlag.End, "Expected Maxlag.End to be not nil") assert.Equalf(t, int64(1400), response.Maxlag.End.Offset, "Expected Maxlag.End.Offset to be 100, not %v", response.Maxlag.End.Offset) stopTestCluster(storageCoordinator, module) } type testset struct { offsets []*protocol.ConsumerOffset brokerOffsets []int64 currentLag uint64 timeNow int64 isLagAlwaysNotZero bool checkIfOffsetsRewind bool checkIfOffsetsStopped bool checkIfOffsetsStalled bool checkIfLagNotDecreasing bool checkIfRecentLagZero bool status protocol.StatusConstant } // This section is the "correctness" proof for the evaluator. Please add lots of tests here, as it's critical that this // code operate properly and give good results every time. // // When adding tests, remember the following things: // 1) The Timestamp fields are in milliseconds, but the timeNow field is in seconds // 2) The tests are performed individually and in sequence. This means that it's possible for multiple rules to be triggered // 3) The status represents what would be returned for this set of offsets when processing rules in sequence // 4) Use this to add tests for offset sets that you think (or know) are producing false results when improving the checks // 5) Tests should be commented with the index number, as well as what they are trying to test and why the expected results are correct // 5) If you change an existing test, there needs to be a good explanation as to why along with the PR var tests = []testset{ // 0 - returns OK because there is zero lag somewhere { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 0}, {2000, 200000, 50}, {3000, 300000, 100}, {4000, 400000, 150}, {5000, 500000, 200}, }, brokerOffsets: []int64{5200}, currentLag: 200, timeNow: 600, isLagAlwaysNotZero: false, checkIfOffsetsRewind: false, checkIfOffsetsStopped: false, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusOK, }, // 1 - same status. does not return true for stop because time since last commit (400s) is equal to the difference (500-100), not greater than { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 0}, {2000, 200000, 50}, {3000, 300000, 100}, {4000, 400000, 150}, {5000, 500000, 200}, }, brokerOffsets: []int64{5200}, currentLag: 200, timeNow: 900, isLagAlwaysNotZero: false, checkIfOffsetsRewind: false, checkIfOffsetsStopped: false, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusOK, }, // 2 - status is now STOP because the time since last commit is great enough (500s), even though lag is zero at the start (fixed due to #290) { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 0}, {2000, 200000, 50}, {3000, 300000, 100}, {4000, 400000, 150}, {5000, 500000, 200}, }, brokerOffsets: []int64{5200}, currentLag: 200, timeNow: 1000, isLagAlwaysNotZero: false, checkIfOffsetsRewind: false, checkIfOffsetsStopped: true, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusStop, }, // 3 - status is STOP when lag is always non-zero as well { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 50}, {2000, 200000, 100}, {3000, 300000, 150}, {4000, 400000, 200}, {5000, 500000, 250}, }, brokerOffsets: []int64{5250}, currentLag: 250, timeNow: 1000, isLagAlwaysNotZero: true, checkIfOffsetsRewind: false, checkIfOffsetsStopped: true, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusStop, }, // 4 - status is OK because of zero lag, but stall is true because the offset is always the same and there is lag (another commit with turn this to stall) { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 0}, {1000, 200000, 50}, {1000, 300000, 100}, {1000, 400000, 150}, {1000, 500000, 200}, }, brokerOffsets: []int64{1200}, currentLag: 200, timeNow: 600, isLagAlwaysNotZero: false, checkIfOffsetsRewind: false, checkIfOffsetsStopped: false, checkIfOffsetsStalled: true, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusOK, }, // 5 - status is now STALL because the lag is always non-zero { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 100}, {1000, 200000, 150}, {1000, 300000, 200}, {1000, 400000, 250}, {1000, 500000, 300}, }, brokerOffsets: []int64{1300}, currentLag: 300, timeNow: 600, isLagAlwaysNotZero: true, checkIfOffsetsRewind: false, checkIfOffsetsStopped: false, checkIfOffsetsStalled: true, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusStall, }, // 6 - status is still STALL even when the lag stays the same { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 100}, {1000, 200000, 100}, {1000, 300000, 100}, {1000, 400000, 100}, {1000, 500000, 100}, }, brokerOffsets: []int64{1100}, currentLag: 100, timeNow: 600, isLagAlwaysNotZero: true, checkIfOffsetsRewind: false, checkIfOffsetsStopped: false, checkIfOffsetsStalled: true, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusStall, }, // 7 - status is REWIND because the offsets go backwards, even though the lag does decrease (rewind is worse) { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 100}, {2000, 200000, 150}, {3000, 300000, 200}, {2000, 400000, 1250}, {4000, 500000, 300}, }, brokerOffsets: []int64{4300}, currentLag: 300, timeNow: 600, isLagAlwaysNotZero: true, checkIfOffsetsRewind: true, checkIfOffsetsStopped: false, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: false, checkIfRecentLagZero: false, status: protocol.StatusRewind, }, // 8 - status is OK because the current lag is 0 (even though the offsets show lag), even though it would be considered stopped due to timestamps { offsets: []*protocol.ConsumerOffset{ {1000, 100000, 50}, {2000, 200000, 100}, {3000, 300000, 150}, {4000, 400000, 200}, {5000, 500000, 250}, }, brokerOffsets: []int64{5250}, currentLag: 0, timeNow: 1000, isLagAlwaysNotZero: true, checkIfOffsetsRewind: false, checkIfOffsetsStopped: true, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusOK, }, // 9 - status is STOP due to timestamps because the current lag is non-zero, even though lag is always zero in offsets, only because there is new data (#290) { offsets: []*protocol.ConsumerOffset{ {792748079, 1512224618356, 0}, {792748080, 1512224619362, 0}, {792748081, 1512224620366, 0}, {792748082, 1512224621367, 0}, {792748083, 1512224622370, 0}, {792748084, 1512224623373, 0}, {792748085, 1512224624378, 0}, {792748086, 1512224625379, 0}, {792748087, 1512224626383, 0}, {792748088, 1512224627383, 0}, {792748089, 1512224628383, 0}, {792748090, 1512224629388, 0}, {792748091, 1512224630391, 0}, {792748092, 1512224631394, 0}, {792748093, 1512224632397, 0}, }, brokerOffsets: []int64{792749024, 792749000, 792748800, 792748600, 792748500}, currentLag: 931, timeNow: 1512224650, isLagAlwaysNotZero: false, checkIfOffsetsRewind: false, checkIfOffsetsStopped: true, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: false, status: protocol.StatusStop, }, // 10 - status is OK, even though it would be stop due to timestamps, as within the recent broker offset window the lag was zero (#303) { offsets: []*protocol.ConsumerOffset{ {792748079, 1512224618356, 0}, {792748080, 1512224619362, 0}, {792748081, 1512224620366, 0}, {792748082, 1512224621367, 0}, {792748083, 1512224622370, 0}, {792748084, 1512224623373, 0}, {792748085, 1512224624378, 0}, {792748086, 1512224625379, 0}, {792748087, 1512224626383, 0}, {792748088, 1512224627383, 0}, {792748089, 1512224628383, 0}, {792748090, 1512224629388, 0}, {792748091, 1512224630391, 0}, {792748092, 1512224631394, 0}, {792748093, 1512224632397, 0}, }, brokerOffsets: []int64{792748094, 792748093, 792748093, 792748093}, currentLag: 1, timeNow: 1512224650, isLagAlwaysNotZero: false, checkIfOffsetsRewind: false, checkIfOffsetsStopped: true, checkIfOffsetsStalled: false, checkIfLagNotDecreasing: true, checkIfRecentLagZero: true, status: protocol.StatusOK, }, } func TestCachingEvaluator_CheckRules(t *testing.T) { for i, testSet := range tests { result := isLagAlwaysNotZero(testSet.offsets) assert.Equalf(t, testSet.isLagAlwaysNotZero, result, "TEST %v: Expected isLagAlwaysNotZero to return %v, not %v", i, testSet.isLagAlwaysNotZero, result) result = checkIfOffsetsRewind(testSet.offsets) assert.Equalf(t, testSet.checkIfOffsetsRewind, result, "TEST %v: Expected checkIfOffsetsRewind to return %v, not %v", i, testSet.checkIfOffsetsRewind, result) result = checkIfOffsetsStopped(testSet.offsets, testSet.timeNow) assert.Equalf(t, testSet.checkIfOffsetsStopped, result, "TEST %v: Expected checkIfOffsetsStopped to return %v, not %v", i, testSet.checkIfOffsetsStopped, result) result = checkIfOffsetsStalled(testSet.offsets) assert.Equalf(t, testSet.checkIfOffsetsStalled, result, "TEST %v: Expected checkIfOffsetsStalled to return %v, not %v", i, testSet.checkIfOffsetsStalled, result) result = checkIfLagNotDecreasing(testSet.offsets) assert.Equalf(t, testSet.checkIfLagNotDecreasing, result, "TEST %v: Expected checkIfLagNotDecreasing to return %v, not %v", i, testSet.checkIfLagNotDecreasing, result) result = checkIfRecentLagZero(testSet.offsets, testSet.brokerOffsets) assert.Equalf(t, testSet.checkIfRecentLagZero, result, "TEST %v: Expected checkIfRecentLagZero to return %v, not %v", i, testSet.checkIfRecentLagZero, result) status := calculatePartitionStatus(testSet.offsets, testSet.brokerOffsets, testSet.currentLag, testSet.timeNow) assert.Equalf(t, testSet.status, status, "TEST %v: Expected calculatePartitionStatus to return %v, not %v", i, testSet.status.String(), status.String()) } } // TODO this test should fail, ie a group should not exist if all its topics are deleted. func TestCachingEvaluator_TopicDeleted(t *testing.T) { storageCoordinator, module := fixtureModule() module.Configure("test", "evaluator.test") module.Start() // Deleting a topic will not delete the consumer group even if the consumer group has no topics storageCoordinator.App.StorageChannel <- &protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteTopic, Cluster: "testcluster", Topic: "testtopic", } time.Sleep(100 * time.Millisecond) evalRequest := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "testgroup", ShowAll: true, } module.GetCommunicationChannel() <- evalRequest evalResponse := <-evalRequest.Reply // The status is returned as 'OK' as the topic has previously existed and // belonged to a group therefore if the group has a recent timestamp in the // consumerMap the evaluator continues as normal but processes no // partitions. assert.Equalf(t, protocol.StatusOK, evalResponse.Status, "Expected status to be OK, not %v", evalResponse.Status.String()) assert.Emptyf(t, evalResponse.Partitions, "Expected no partitions to be returned") assert.Equalf(t, float32(0.0), evalResponse.Complete, "Expected 'Complete' to be 0.0") } burrow-1.2.1/core/internal/evaluator/coordinator.go000066400000000000000000000143371343357346000224730ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package evaluator - Group evaluation subsystem. // The evaluator subsystem is responsible for fetching group information from the storage subsystem and calculating the // group's status based on that. It responds to EvaluatorRequest objects that are send via a channel, and replies with // a ConsumerGroupStatus. // // Modules // // Currently, only one module is provided: // // * caching - Evaluate a consumer group and cache the results in memory for a short period of time package evaluator import ( "errors" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // Module is responsible for answering requests to evaluate the status of a consumer group. It fetches offset // information from the storage subsystem and transforms that into a protocol.ConsumerGroupStatus response. It conforms // to the overall protocol.Module interface, but it adds a func to fetch the channel that the module is listening on for // requests, so that requests can be forwarded to it by the coordinator type Module interface { protocol.Module GetCommunicationChannel() chan *protocol.EvaluatorRequest } // Coordinator manages a single evaluator module (only one module is supported at this time), making sure it is // configured, started, and stopped at the appropriate time. It is also responsible for listening to the // EvaluatorChannel that is provided in the application context and forwarding those requests to the evaluator module. // If no evaluator module has been configured explicitly, the coordinator starts the caching module as a default. type Coordinator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger quitChannel chan struct{} modules map[string]protocol.Module } // getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there // is any error, it will panic with an appropriate message describing the problem. func getModuleForClass(app *protocol.ApplicationContext, moduleName string, className string) protocol.Module { switch className { case "caching": return &CachingEvaluator{ App: app, Log: app.Logger.With( zap.String("type", "module"), zap.String("coordinator", "evaluator"), zap.String("class", className), zap.String("name", moduleName), ), } default: panic("Unknown evaluator className provided: " + className) } } // Configure is called to create the configured evaluator module and call its Configure func to validate the // configuration and set it up. The coordinator will panic is more than one module is configured, and if no modules have // been configured, it will set up a default caching evaluator module. If there are any problems, it is expected that // this func will panic with a descriptive error message, as configuration failures are not recoverable errors. func (ec *Coordinator) Configure() { ec.Log.Info("configuring") ec.quitChannel = make(chan struct{}) ec.modules = make(map[string]protocol.Module) modules := viper.GetStringMap("evaluator") switch len(modules) { case 0: // Create a default module viper.Set("evaluator.default.class-name", "caching") modules = viper.GetStringMap("evaluator") case 1: // Have one module. Just continue break default: panic("Only one evaluator module must be configured") } // Create all configured evaluator modules, add to list of evaluators for name := range modules { configRoot := "evaluator." + name module := getModuleForClass(ec.App, name, viper.GetString(configRoot+".class-name")) module.Configure(name, configRoot) ec.modules[name] = module } } // Start calls the evaluator module's underlying Start func. If the module Start returns an error, this func stops // immediately and returns that error to the caller. // // We also start a request forwarder goroutine. This listens to the EvaluatorChannel that is provided in the application // context that all modules receive, and forwards those requests to the evaluator modules. At the present time, the // evaluator only supports one module, so this is a simple "accept and forward". func (ec *Coordinator) Start() error { ec.Log.Info("starting") // Start Evaluator modules err := helpers.StartCoordinatorModules(ec.modules) if err != nil { return errors.New("Error starting evaluator module: " + err.Error()) } // Start request forwarder go func() { // We only support 1 module right now, so only send to that module var channel chan *protocol.EvaluatorRequest for _, module := range ec.modules { channel = module.(Module).GetCommunicationChannel() } for { select { case request := <-ec.App.EvaluatorChannel: // Yes, this forwarder is silly. However, in the future we want to support multiple evaluator modules // concurrently. However, that will require implementing a router that properly handles requests and // makes sure that only 1 evaluator responds channel <- request case <-ec.quitChannel: return } } }() return nil } // Stop calls the configured evaluator module's underlying Stop func. It is expected that the module Stop will not // return until the module has been completely stopped. While an error can be returned, this func always returns no // error, as a failure during stopping is not a critical failure func (ec *Coordinator) Stop() error { ec.Log.Info("stopping") close(ec.quitChannel) // The individual storage modules can choose whether or not to implement a wait in the Stop routine helpers.StopCoordinatorModules(ec.modules) return nil } burrow-1.2.1/core/internal/evaluator/coordinator_test.go000066400000000000000000000101341343357346000235210ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package evaluator import ( "github.com/stretchr/testify/assert" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) func fixtureCoordinator() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("evaluator.test.class-name", "caching") viper.Set("evaluator.test.expire-cache", 30) viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) return &coordinator } func TestCoordinator_ImplementsCoordinator(t *testing.T) { assert.Implements(t, (*protocol.Coordinator)(nil), new(Coordinator)) } func TestCoordinator_Configure(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) } func TestCoordinator_Configure_NoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Reset() viper.Set("cluster.test.class-name", "kafka") viper.Set("cluster.test.servers", []string{"broker1.example.com:1234"}) coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) } func TestCoordinator_Configure_TwoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("evaluator.anothertest.class-name", "caching") viper.Set("evaluator.anothertest.expire-cache", 30) assert.Panics(t, coordinator.Configure, "Expected panic") } func TestCoordinator_Start(t *testing.T) { evaluatorCoordinator, storageCoordinator := StorageAndEvaluatorCoordinatorsWithOffsets() // Best is to test a request that we know the response to request := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "testgroup", ShowAll: true, } evaluatorCoordinator.App.EvaluatorChannel <- request response := <-request.Reply assert.Equalf(t, protocol.StatusOK, response.Status, "Expected status to be OK, not %v", response.Status.String()) assert.Equalf(t, float32(1.0), response.Complete, "Expected complete to be 1.0, not %v", response.Complete) assert.Equalf(t, 1, response.TotalPartitions, "Expected total_partitions to be 1, not %v", response.TotalPartitions) assert.Equalf(t, uint64(2421), response.TotalLag, "Expected total_lag to be 2421, not %v", response.TotalLag) assert.Equalf(t, "testcluster", response.Cluster, "Expected cluster to be testcluster, not %v", response.Cluster) assert.Equalf(t, "testgroup", response.Group, "Expected group to be testgroup, not %v", response.Group) assert.Lenf(t, response.Partitions, 1, "Expected 1 partition status objects, not %v", len(response.Partitions)) evaluatorCoordinator.Stop() storageCoordinator.Stop() } func TestCoordinator_MultipleRequests(t *testing.T) { evaluatorCoordinator, storageCoordinator := StorageAndEvaluatorCoordinatorsWithOffsets() // This test is really just to check and make sure the evaluator can handle multiple requests without deadlock for i := 0; i < 10; i++ { request := &protocol.EvaluatorRequest{ Reply: make(chan *protocol.ConsumerGroupStatus), Cluster: "testcluster", Group: "testgroup", ShowAll: true, } evaluatorCoordinator.App.EvaluatorChannel <- request response := <-request.Reply assert.Equalf(t, protocol.StatusOK, response.Status, "Expected status to be OK, not %v", response.Status.String()) } // Best is to test a request that we know the response to evaluatorCoordinator.Stop() storageCoordinator.Stop() } burrow-1.2.1/core/internal/evaluator/fixtures.go000066400000000000000000000030601343357346000220100ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package evaluator import ( "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/storage" "github.com/linkedin/Burrow/core/protocol" ) // StorageAndEvaluatorCoordinatorsWithOffsets sets up a Coordinator with a single caching module defined. In order to do // this, it also calls the storage subsystem fixture to get a configured storage.Coordinator with offsets for a test // cluster and group. This func should never be called in normal code. It is only provided to facilitate testing by // other subsystems. func StorageAndEvaluatorCoordinatorsWithOffsets() (*Coordinator, *storage.Coordinator) { storageCoordinator := storage.CoordinatorWithOffsets() evaluatorCoordinator := Coordinator{ Log: zap.NewNop(), } evaluatorCoordinator.App = storageCoordinator.App evaluatorCoordinator.App.EvaluatorChannel = make(chan *protocol.EvaluatorRequest) viper.Set("evaluator.test.class-name", "caching") viper.Set("evaluator.test.expire-cache", 30) evaluatorCoordinator.Configure() evaluatorCoordinator.Start() return &evaluatorCoordinator, storageCoordinator } burrow-1.2.1/core/internal/helpers/000077500000000000000000000000001343357346000172515ustar00rootroot00000000000000burrow-1.2.1/core/internal/helpers/coordinators.go000066400000000000000000000067451343357346000223220ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package helpers - Common utilities. // The helpers subsystem provides common utilities that can be used by all subsystems. This includes utilities for // coordinators to start and stop modules, as well as Kafka and Zookeeper client implementations. There are also a // number of mocks that are provided for testing purposes only, and should not be used in normal code. package helpers import ( "regexp" "time" "github.com/stretchr/testify/mock" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) // StartCoordinatorModules is a helper func for coordinators to start a list of modules. Given a map of protocol.Module, // it calls the Start func on each one. If any module returns an error, it immediately stops and returns that error func StartCoordinatorModules(modules map[string]protocol.Module) error { // Start all the modules, returning an error if any fail to start for _, module := range modules { err := module.Start() if err != nil { return err } } return nil } // StopCoordinatorModules is a helper func for coordinators to stop a list of modules. Given a map of protocol.Module, // it calls the Stop func on each one. Any errors that are returned are ignored. func StopCoordinatorModules(modules map[string]protocol.Module) { // Stop all the modules passed in for _, module := range modules { module.Stop() } } // MockModule is a mock of protocol.Module that also satisfies the various subsystem Module variants, and is used in // tests. It should never be used in the normal code. type MockModule struct { mock.Mock } // Configure mocks the protocol.Module Configure func func (m *MockModule) Configure(name string, configRoot string) { m.Called(name, configRoot) } // Start mocks the protocol.Module Start func func (m *MockModule) Start() error { args := m.Called() return args.Error(0) } // Stop mocks the protocol.Module Stop func func (m *MockModule) Stop() error { args := m.Called() return args.Error(0) } // GetName mocks the notifier.Module GetName func func (m *MockModule) GetName() string { args := m.Called() return args.String(0) } // GetGroupWhitelist mocks the notifier.Module GetGroupWhitelist func func (m *MockModule) GetGroupWhitelist() *regexp.Regexp { args := m.Called() return args.Get(0).(*regexp.Regexp) } // GetGroupBlacklist mocks the notifier.Module GetGroupBlacklist func func (m *MockModule) GetGroupBlacklist() *regexp.Regexp { args := m.Called() return args.Get(0).(*regexp.Regexp) } // GetLogger mocks the notifier.Module GetLogger func func (m *MockModule) GetLogger() *zap.Logger { args := m.Called() return args.Get(0).(*zap.Logger) } // AcceptConsumerGroup mocks the notifier.Module AcceptConsumerGroup func func (m *MockModule) AcceptConsumerGroup(status *protocol.ConsumerGroupStatus) bool { args := m.Called(status) return args.Bool(0) } // Notify mocks the notifier.Module Notify func func (m *MockModule) Notify(status *protocol.ConsumerGroupStatus, eventID string, startTime time.Time, stateGood bool) { m.Called(status, eventID, startTime, stateGood) } burrow-1.2.1/core/internal/helpers/coordinators_test.go000066400000000000000000000034231343357346000233470ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "errors" "testing" "github.com/stretchr/testify/assert" "github.com/linkedin/Burrow/core/protocol" ) func TestStartCoordinatorModules(t *testing.T) { mock1 := &MockModule{} mock2 := &MockModule{} modules := map[string]protocol.Module{ "mock1": mock1, "mock2": mock2, } mock1.On("Start").Return(nil) mock2.On("Start").Return(nil) err := StartCoordinatorModules(modules) assert.Nil(t, err, "Expected error to be nil") mock1.AssertExpectations(t) mock2.AssertExpectations(t) } func TestStartCoordinatorModules_Error(t *testing.T) { mock1 := &MockModule{} mock2 := &MockModule{} modules := map[string]protocol.Module{ "mock1": mock1, "mock2": mock2, } mock1.On("Start").Return(nil) mock2.On("Start").Return(errors.New("bad start")) err := StartCoordinatorModules(modules) assert.NotNil(t, err, "Expected error to be nil") // Can't assert expectations, as it's possible that mock1 won't be called due to non-deterministic ordering of range } func TestStopCoordinatorModules(t *testing.T) { mock1 := &MockModule{} mock2 := &MockModule{} modules := map[string]protocol.Module{ "mock1": mock1, "mock2": mock2, } mock1.On("Stop").Return(nil) mock2.On("Stop").Return(nil) StopCoordinatorModules(modules) mock1.AssertExpectations(t) mock2.AssertExpectations(t) } burrow-1.2.1/core/internal/helpers/sarama.go000066400000000000000000000466241343357346000210600ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "crypto/tls" "crypto/x509" "io/ioutil" "github.com/Shopify/sarama" "github.com/spf13/viper" "github.com/stretchr/testify/mock" ) var kafkaVersions = map[string]sarama.KafkaVersion{ "": sarama.V0_10_2_0, "0.8.0": sarama.V0_8_2_0, "0.8.1": sarama.V0_8_2_1, "0.8.2": sarama.V0_8_2_2, "0.8": sarama.V0_8_2_0, "0.9.0.0": sarama.V0_9_0_0, "0.9.0.1": sarama.V0_9_0_1, "0.9.0": sarama.V0_9_0_0, "0.9": sarama.V0_9_0_0, "0.10.0.0": sarama.V0_10_0_0, "0.10.0.1": sarama.V0_10_0_1, "0.10.0": sarama.V0_10_0_0, "0.10.1.0": sarama.V0_10_1_0, "0.10.1": sarama.V0_10_1_0, "0.10.2.0": sarama.V0_10_2_0, "0.10.2.1": sarama.V0_10_2_0, "0.10.2": sarama.V0_10_2_0, "0.10": sarama.V0_10_0_0, "0.11.0.1": sarama.V0_11_0_0, "0.11.0.2": sarama.V0_11_0_0, "0.11.0": sarama.V0_11_0_0, "1.0.0": sarama.V1_0_0_0, "1.1.0": sarama.V1_1_0_0, "1.1.1": sarama.V1_1_0_0, "2.0.0": sarama.V2_0_0_0, "2.0.1": sarama.V2_0_0_0, "2.1.0": sarama.V2_1_0_0, } func parseKafkaVersion(kafkaVersion string) sarama.KafkaVersion { version, ok := kafkaVersions[string(kafkaVersion)] if !ok { panic("Unknown Kafka Version: " + kafkaVersion) } return version } // GetSaramaConfigFromClientProfile takes the name of a client-profile configuration entry and returns a sarama.Config // object that can be used to create a Sarama client with the specified configuration. This includes the Kafka version, // client ID, TLS, and SASL configs. If there is any error in the configuration, such as a bad TLS certificate file, // this func will panic as it is normally called when configuring modules. func GetSaramaConfigFromClientProfile(profileName string) *sarama.Config { // Set config root and defaults configRoot := "client-profile." + profileName if (profileName != "") && (!viper.IsSet("client-profile." + profileName)) { panic("unknown client-profile '" + profileName + "'") } viper.SetDefault(configRoot+".client-id", "burrow-lagchecker") viper.SetDefault(configRoot+".kafka-version", "0.8") saramaConfig := sarama.NewConfig() saramaConfig.ClientID = viper.GetString(configRoot + ".client-id") saramaConfig.Version = parseKafkaVersion(viper.GetString(configRoot + ".kafka-version")) saramaConfig.Consumer.Return.Errors = true // Configure TLS if enabled if viper.IsSet(configRoot + ".tls") { tlsName := viper.GetString(configRoot + ".tls") saramaConfig.Net.TLS.Enable = true certFile := viper.GetString("tls." + tlsName + ".certfile") keyFile := viper.GetString("tls." + tlsName + ".keyfile") caFile := viper.GetString("tls." + tlsName + ".cafile") if certFile == "" || keyFile == "" || caFile == "" { saramaConfig.Net.TLS.Config = &tls.Config{} } else { caCert, err := ioutil.ReadFile(caFile) if err != nil { panic("cannot read TLS CA file: " + err.Error()) } cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { panic("cannot read TLS certificate or key file: " + err.Error()) } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) saramaConfig.Net.TLS.Config = &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caCertPool, } saramaConfig.Net.TLS.Config.BuildNameToCertificate() } saramaConfig.Net.TLS.Config.InsecureSkipVerify = viper.GetBool("tls." + tlsName + ".noverify") } // Configure SASL if enabled if viper.IsSet(configRoot + ".sasl") { saslName := viper.GetString(configRoot + ".sasl") saramaConfig.Net.SASL.Enable = true saramaConfig.Net.SASL.Handshake = viper.GetBool("sasl." + saslName + ".handshake-first") saramaConfig.Net.SASL.User = viper.GetString("sasl." + saslName + ".username") saramaConfig.Net.SASL.Password = viper.GetString("sasl." + saslName + ".password") } return saramaConfig } // SaramaClient is an internal interface to the sarama.Client. We use our own interface because while sarama.Client is // an interface, sarama.Broker is not. This makes it difficult to test code which uses the Broker objects. This // interface operates in the same way, with the addition of an interface function for creating consumers on the client. type SaramaClient interface { // Config returns the Config struct of the client. This struct should not be altered after it has been created. Config() *sarama.Config // Brokers returns the current set of active brokers as retrieved from cluster metadata. Brokers() []SaramaBroker // Topics returns the set of available topics as retrieved from cluster metadata. Topics() ([]string, error) // Partitions returns the sorted list of all partition IDs for the given topic. Partitions(topic string) ([]int32, error) // WritablePartitions returns the sorted list of all writable partition IDs for the given topic, where "writable" // means "having a valid leader accepting writes". WritablePartitions(topic string) ([]int32, error) // Leader returns the broker object that is the leader of the current topic/partition, as determined by querying the // cluster metadata. Leader(topic string, partitionID int32) (SaramaBroker, error) // Replicas returns the set of all replica IDs for the given partition. Replicas(topic string, partitionID int32) ([]int32, error) // InSyncReplicas returns the set of all in-sync replica IDs for the given partition. In-sync replicas are replicas // which are fully caught up with the partition leader. InSyncReplicas(topic string, partitionID int32) ([]int32, error) // RefreshMetadata takes a list of topics and queries the cluster to refresh the available metadata for those topics. // If no topics are provided, it will refresh metadata for all topics. RefreshMetadata(topics ...string) error // GetOffset queries the cluster to get the most recent available offset at the given time (in milliseconds) on the // topic/partition combination. Time should be OffsetOldest for the earliest available offset, OffsetNewest for the // offset of the message that will be produced next, or a time. GetOffset(topic string, partitionID int32, time int64) (int64, error) // Coordinator returns the coordinating broker for a consumer group. It will return a locally cached value if it's // available. You can call RefreshCoordinator to update the cached value. This function only works on Kafka 0.8.2 and // higher. Coordinator(consumerGroup string) (SaramaBroker, error) // RefreshCoordinator retrieves the coordinator for a consumer group and stores it in local cache. This function only // works on Kafka 0.8.2 and higher. RefreshCoordinator(consumerGroup string) error // Close shuts down all broker connections managed by this client. It is required to call this function before a client // object passes out of scope, as it will otherwise leak memory. You must close any Producers or Consumers using a // client before you close the client. Close() error // Closed returns true if the client has already had Close called on it Closed() bool // NewConsumerFromClient creates a new consumer using the given client. It is still necessary to call Close() on the // underlying client when shutting down this consumer. NewConsumerFromClient() (sarama.Consumer, error) } // BurrowSaramaClient is an implementation of the SaramaClient interface for use in Burrow modules type BurrowSaramaClient struct { Client sarama.Client } // Config returns the Config struct of the client. This struct should not be altered after it has been created. func (c *BurrowSaramaClient) Config() *sarama.Config { return c.Client.Config() } // Brokers returns the current set of active brokers as retrieved from cluster metadata. func (c *BurrowSaramaClient) Brokers() []SaramaBroker { brokers := c.Client.Brokers() shimBrokers := make([]SaramaBroker, len(brokers)) for i, broker := range brokers { shimBrokers[i] = &BurrowSaramaBroker{broker} } return shimBrokers } // Topics returns the set of available topics as retrieved from cluster metadata. func (c *BurrowSaramaClient) Topics() ([]string, error) { return c.Client.Topics() } // Partitions returns the sorted list of all partition IDs for the given topic. func (c *BurrowSaramaClient) Partitions(topic string) ([]int32, error) { return c.Client.Partitions(topic) } // WritablePartitions returns the sorted list of all writable partition IDs for the given topic, where "writable" // means "having a valid leader accepting writes". func (c *BurrowSaramaClient) WritablePartitions(topic string) ([]int32, error) { return c.Client.WritablePartitions(topic) } // Leader returns the broker object that is the leader of the current topic/partition, as determined by querying the // cluster metadata. func (c *BurrowSaramaClient) Leader(topic string, partitionID int32) (SaramaBroker, error) { broker, err := c.Client.Leader(topic, partitionID) var shimBroker *BurrowSaramaBroker if broker != nil { shimBroker = &BurrowSaramaBroker{broker} } return shimBroker, err } // Replicas returns the set of all replica IDs for the given partition. func (c *BurrowSaramaClient) Replicas(topic string, partitionID int32) ([]int32, error) { return c.Client.Replicas(topic, partitionID) } // InSyncReplicas returns the set of all in-sync replica IDs for the given partition. In-sync replicas are replicas // which are fully caught up with the partition leader. func (c *BurrowSaramaClient) InSyncReplicas(topic string, partitionID int32) ([]int32, error) { return c.Client.InSyncReplicas(topic, partitionID) } // RefreshMetadata takes a list of topics and queries the cluster to refresh the available metadata for those topics. // If no topics are provided, it will refresh metadata for all topics. func (c *BurrowSaramaClient) RefreshMetadata(topics ...string) error { return c.Client.RefreshMetadata(topics...) } // GetOffset queries the cluster to get the most recent available offset at the given time (in milliseconds) on the // topic/partition combination. Time should be OffsetOldest for the earliest available offset, OffsetNewest for the // offset of the message that will be produced next, or a time. func (c *BurrowSaramaClient) GetOffset(topic string, partitionID int32, time int64) (int64, error) { return c.Client.GetOffset(topic, partitionID, time) } // Coordinator returns the coordinating broker for a consumer group. It will return a locally cached value if it's // available. You can call RefreshCoordinator to update the cached value. This function only works on Kafka 0.8.2 and // higher. func (c *BurrowSaramaClient) Coordinator(consumerGroup string) (SaramaBroker, error) { broker, err := c.Client.Coordinator(consumerGroup) var shimBroker *BurrowSaramaBroker if broker != nil { shimBroker = &BurrowSaramaBroker{broker} } return shimBroker, err } // RefreshCoordinator retrieves the coordinator for a consumer group and stores it in local cache. This function only // works on Kafka 0.8.2 and higher. func (c *BurrowSaramaClient) RefreshCoordinator(consumerGroup string) error { return c.Client.RefreshCoordinator(consumerGroup) } // Close shuts down all broker connections managed by this client. It is required to call this function before a client // object passes out of scope, as it will otherwise leak memory. You must close any Producers or Consumers using a // client before you close the client. func (c *BurrowSaramaClient) Close() error { return c.Client.Close() } // Closed returns true if the client has already had Close called on it func (c *BurrowSaramaClient) Closed() bool { return c.Client.Closed() } // NewConsumerFromClient creates a new consumer using the given client. It is still necessary to call Close() on the // underlying client when shutting down this consumer. func (c *BurrowSaramaClient) NewConsumerFromClient() (sarama.Consumer, error) { return sarama.NewConsumerFromClient(c.Client) } // SaramaBroker is an internal interface on the sarama.Broker struct. It is used with the SaramaClient interface in // order to provide a fully testable interface for the pieces of Sarama that are used inside Burrow. Currently, this // interface only defines the methods that Burrow is using. It should not be considered a complete interface for // sarama.Broker type SaramaBroker interface { // ID returns the broker ID retrieved from Kafka's metadata, or -1 if that is not known. ID() int32 // Close closes the connection associated with the broker Close() error // GetAvailableOffsets sends an OffsetRequest to the broker and returns the OffsetResponse that was received GetAvailableOffsets(*sarama.OffsetRequest) (*sarama.OffsetResponse, error) } // BurrowSaramaBroker is an implementation of the SaramaBroker interface that is used with SaramaClient type BurrowSaramaBroker struct { broker *sarama.Broker } // ID returns the broker ID retrieved from Kafka's metadata, or -1 if that is not known. func (b *BurrowSaramaBroker) ID() int32 { return b.broker.ID() } // Close closes the connection associated with the broker func (b *BurrowSaramaBroker) Close() error { return b.broker.Close() } // GetAvailableOffsets sends an OffsetRequest to the broker and returns the OffsetResponse that was received func (b *BurrowSaramaBroker) GetAvailableOffsets(request *sarama.OffsetRequest) (*sarama.OffsetResponse, error) { return b.broker.GetAvailableOffsets(request) } // MockSaramaClient is a mock of SaramaClient. It is used in tests by multiple packages. It should never be used in the // normal code. type MockSaramaClient struct { mock.Mock } // Config mocks SaramaClient.Config func (m *MockSaramaClient) Config() *sarama.Config { args := m.Called() return args.Get(0).(*sarama.Config) } // Brokers mocks SaramaClient.Brokers func (m *MockSaramaClient) Brokers() []SaramaBroker { args := m.Called() return args.Get(0).([]SaramaBroker) } // Topics mocks SaramaClient.Topics func (m *MockSaramaClient) Topics() ([]string, error) { args := m.Called() return args.Get(0).([]string), args.Error(1) } // Partitions mocks SaramaClient.Partitions func (m *MockSaramaClient) Partitions(topic string) ([]int32, error) { args := m.Called(topic) return args.Get(0).([]int32), args.Error(1) } // WritablePartitions mocks SaramaClient.WritablePartitions func (m *MockSaramaClient) WritablePartitions(topic string) ([]int32, error) { args := m.Called(topic) return args.Get(0).([]int32), args.Error(1) } // Leader mocks SaramaClient.Leader func (m *MockSaramaClient) Leader(topic string, partitionID int32) (SaramaBroker, error) { args := m.Called(topic, partitionID) return args.Get(0).(SaramaBroker), args.Error(1) } // Replicas mocks SaramaClient.Replicas func (m *MockSaramaClient) Replicas(topic string, partitionID int32) ([]int32, error) { args := m.Called(topic, partitionID) return args.Get(0).([]int32), args.Error(1) } // InSyncReplicas mocks SaramaClient.InSyncReplicas func (m *MockSaramaClient) InSyncReplicas(topic string, partitionID int32) ([]int32, error) { args := m.Called(topic, partitionID) return args.Get(0).([]int32), args.Error(1) } // RefreshMetadata mocks SaramaClient.RefreshMetadata func (m *MockSaramaClient) RefreshMetadata(topics ...string) error { if len(topics) > 0 { args := m.Called([]interface{}{topics}...) return args.Error(0) } args := m.Called() return args.Error(0) } // GetOffset mocks SaramaClient.GetOffset func (m *MockSaramaClient) GetOffset(topic string, partitionID int32, time int64) (int64, error) { args := m.Called(topic, partitionID, time) return args.Get(0).(int64), args.Error(1) } // Coordinator mocks SaramaClient.Coordinator func (m *MockSaramaClient) Coordinator(consumerGroup string) (SaramaBroker, error) { args := m.Called(consumerGroup) return args.Get(0).(SaramaBroker), args.Error(1) } // RefreshCoordinator mocks SaramaClient.RefreshCoordinator func (m *MockSaramaClient) RefreshCoordinator(consumerGroup string) error { args := m.Called(consumerGroup) return args.Error(0) } // Close mocks SaramaClient.Close func (m *MockSaramaClient) Close() error { args := m.Called() return args.Error(0) } // Closed mocks SaramaClient.Closed func (m *MockSaramaClient) Closed() bool { args := m.Called() return args.Bool(0) } // NewConsumerFromClient mocks SaramaClient.NewConsumerFromClient func (m *MockSaramaClient) NewConsumerFromClient() (sarama.Consumer, error) { args := m.Called() return args.Get(0).(sarama.Consumer), args.Error(1) } // MockSaramaBroker is a mock of SaramaBroker. It is used in tests by multiple packages. It should never be used in the // normal code. type MockSaramaBroker struct { mock.Mock } // ID mocks SaramaBroker.ID func (m *MockSaramaBroker) ID() int32 { args := m.Called() return args.Get(0).(int32) } // Close mocks SaramaBroker.Close func (m *MockSaramaBroker) Close() error { args := m.Called() return args.Error(0) } // GetAvailableOffsets mocks SaramaBroker.GetAvailableOffsets func (m *MockSaramaBroker) GetAvailableOffsets(request *sarama.OffsetRequest) (*sarama.OffsetResponse, error) { args := m.Called(request) return args.Get(0).(*sarama.OffsetResponse), args.Error(1) } // MockSaramaConsumer is a mock of sarama.Consumer. It is used in tests by multiple packages. It should never be used // in the normal code. type MockSaramaConsumer struct { mock.Mock } // Topics mocks sarama.Consumer.Topics func (m *MockSaramaConsumer) Topics() ([]string, error) { args := m.Called() return args.Get(0).([]string), args.Error(1) } // Partitions mocks sarama.Consumer.Partitions func (m *MockSaramaConsumer) Partitions(topic string) ([]int32, error) { args := m.Called(topic) return args.Get(0).([]int32), args.Error(1) } // ConsumePartition mocks sarama.Consumer.ConsumePartition func (m *MockSaramaConsumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) { args := m.Called(topic, partition, offset) return args.Get(0).(sarama.PartitionConsumer), args.Error(1) } // HighWaterMarks mocks sarama.Consumer.HighWaterMarks func (m *MockSaramaConsumer) HighWaterMarks() map[string]map[int32]int64 { args := m.Called() return args.Get(0).(map[string]map[int32]int64) } // Close mocks sarama.Consumer.Close func (m *MockSaramaConsumer) Close() error { args := m.Called() return args.Error(0) } // MockSaramaPartitionConsumer is a mock of sarama.PartitionConsumer. It is used in tests by multiple packages. It // should never be used in the normal code. type MockSaramaPartitionConsumer struct { mock.Mock } // AsyncClose mocks sarama.PartitionConsumer.AsyncClose func (m *MockSaramaPartitionConsumer) AsyncClose() { m.Called() } // Close mocks sarama.PartitionConsumer.Close func (m *MockSaramaPartitionConsumer) Close() error { args := m.Called() return args.Error(0) } // Messages mocks sarama.PartitionConsumer.Messages func (m *MockSaramaPartitionConsumer) Messages() <-chan *sarama.ConsumerMessage { args := m.Called() return args.Get(0).(<-chan *sarama.ConsumerMessage) } // Errors mocks sarama.PartitionConsumer.Errors func (m *MockSaramaPartitionConsumer) Errors() <-chan *sarama.ConsumerError { args := m.Called() return args.Get(0).(<-chan *sarama.ConsumerError) } // HighWaterMarkOffset mocks sarama.PartitionConsumer.HighWaterMarkOffset func (m *MockSaramaPartitionConsumer) HighWaterMarkOffset() int64 { args := m.Called() return args.Get(0).(int64) } burrow-1.2.1/core/internal/helpers/sarama_test.go000066400000000000000000000026611343357346000221100ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "testing" "github.com/Shopify/sarama" "github.com/stretchr/testify/assert" ) func TestBurrowSaramaClient_ImplementsSaramaClient(t *testing.T) { assert.Implements(t, (*SaramaClient)(nil), new(BurrowSaramaClient)) } func TestMockSaramaClient_ImplementsSaramaClient(t *testing.T) { assert.Implements(t, (*SaramaClient)(nil), new(MockSaramaClient)) } func TestBurrowSaramaBroker_ImplementsSaramaBroker(t *testing.T) { assert.Implements(t, (*SaramaBroker)(nil), new(BurrowSaramaBroker)) } func TestMockSaramaBroker_ImplementsSaramaBroker(t *testing.T) { assert.Implements(t, (*SaramaBroker)(nil), new(MockSaramaBroker)) } func TestMockSaramaConsumer_ImplementsSaramaConsumer(t *testing.T) { assert.Implements(t, (*sarama.Consumer)(nil), new(MockSaramaConsumer)) } func TestMockSaramaPartitionConsumer_ImplementsSaramaPartitionConsumer(t *testing.T) { assert.Implements(t, (*sarama.PartitionConsumer)(nil), new(MockSaramaPartitionConsumer)) } burrow-1.2.1/core/internal/helpers/storage.go000066400000000000000000000020551343357346000212460ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "time" "github.com/linkedin/Burrow/core/protocol" ) // TimeoutSendStorageRequest is a helper func for sending a protocol.StorageRequest to a channel with a timeout, // specified in seconds. If the request is sent, return true. Otherwise, if the timeout is hit, return false. func TimeoutSendStorageRequest(storageChannel chan *protocol.StorageRequest, request *protocol.StorageRequest, maxTime int) bool { timeout := time.After(time.Duration(maxTime) * time.Second) select { case storageChannel <- request: return true case <-timeout: return false } } burrow-1.2.1/core/internal/helpers/storage_test.go000066400000000000000000000030051343357346000223010ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "github.com/stretchr/testify/assert" "testing" "time" "github.com/linkedin/Burrow/core/protocol" ) func TestTimeoutSendStorageRequest(t *testing.T) { storageChannel := make(chan *protocol.StorageRequest) storageRequest := &protocol.StorageRequest{} go TimeoutSendStorageRequest(storageChannel, storageRequest, 1) // Sleep for 0.5 seconds before reading. There should be a storage request waiting time.Sleep(500 * time.Millisecond) readRequest := <-storageChannel assert.Equal(t, storageRequest, readRequest, "Expected to receive the same storage request") } func TestTimeoutSendStorageRequest_Timeout(t *testing.T) { storageChannel := make(chan *protocol.StorageRequest) storageRequest := &protocol.StorageRequest{} go TimeoutSendStorageRequest(storageChannel, storageRequest, 1) // Sleep for 1.5 seconds before reading. There should be nothing waiting time.Sleep(1500 * time.Millisecond) select { case <-storageChannel: assert.Fail(t, "Expected to not receive storage request after timeout") default: } } burrow-1.2.1/core/internal/helpers/time.go000066400000000000000000000065771343357346000205550ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "github.com/stretchr/testify/mock" "time" ) // Ticker is a generic interface for a channel that delivers `ticks' of a clock at intervals. type Ticker interface { // Start sending ticks over the channel Start() // Stop sending ticks over the channel Stop() // Return the channel that ticks will be sent over GetChannel() <-chan time.Time } // PausableTicker is an implementation of Ticker which can be stopped and restarted without changing the underlying // channel. This is useful for cases where you may need to stop performing actions for a while (such as sending // notifications), but you do not want to tear down everything. type PausableTicker struct { channel chan time.Time duration time.Duration ticker *time.Ticker quitChannel chan struct{} } // NewPausableTicker returns a Ticker that has not yet been started, but the channel is ready to use. This ticker can be // started and stopped multiple times without needing to swap the ticker channel func NewPausableTicker(d time.Duration) Ticker { return &PausableTicker{ channel: make(chan time.Time), duration: d, ticker: nil, } } // Start begins sending ticks over the channel at the interval that has already been configured. If the ticker is // already sending ticks, this func has no effect. func (ticker *PausableTicker) Start() { if ticker.ticker != nil { // Don't restart a ticker that's already running return } // Channel to be able to close the goroutine ticker.quitChannel = make(chan struct{}) // Start the ticker ticker.ticker = time.NewTicker(ticker.duration) // This goroutine will forward the ticker ticks to our exposed channel go func(tickerChan <-chan time.Time, quitChan chan struct{}) { for { select { case tick := <-tickerChan: ticker.channel <- tick case <-quitChan: return } } }(ticker.ticker.C, ticker.quitChannel) } // Stop stops ticks from being sent over the channel. If the ticker is not currently sending ticks, this func has no // effect func (ticker *PausableTicker) Stop() { if ticker.ticker == nil { // Don't stop an already stopped ticker return } // Stop the underlying ticker ticker.ticker.Stop() ticker.ticker = nil // Tell our goroutine to quit close(ticker.quitChannel) } // GetChannel returns the channel over which ticks will be sent. This channel can be used over multiple Start/Stop // cycles, and will not be closed. func (ticker *PausableTicker) GetChannel() <-chan time.Time { return ticker.channel } // MockTicker is a mock Ticker interface that can be used for testing. It should not be used in normal code. type MockTicker struct { mock.Mock } // Start mocks Ticker.Start func (m *MockTicker) Start() { m.Called() } // Stop mocks Ticker.Stop func (m *MockTicker) Stop() { m.Called() } // GetChannel mocks Ticker.GetChannel func (m *MockTicker) GetChannel() <-chan time.Time { args := m.Called() return args.Get(0).(<-chan time.Time) } burrow-1.2.1/core/internal/helpers/time_test.go000066400000000000000000000044271343357346000216040ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "github.com/stretchr/testify/assert" "sync" "testing" "time" ) func TestPausableTicker_ImplementsTicker(t *testing.T) { assert.Implements(t, (*Ticker)(nil), new(PausableTicker)) } func TestPausableTicker_New(t *testing.T) { ticker := NewPausableTicker(5 * time.Millisecond) assert.Implements(t, (*Ticker)(nil), ticker) // We shouldn't get any events across the channel quitChan := make(chan struct{}) channel := ticker.GetChannel() go func() { select { case <-channel: assert.Fail(t, "Expected to receive no event on ticker channel") case <-quitChan: break } }() time.Sleep(25 * time.Millisecond) close(quitChan) } func TestPausableTicker_StartStop(t *testing.T) { ticker := NewPausableTicker(20 * time.Millisecond) ticker.Start() numEvents := 0 quitChan := make(chan struct{}) channel := ticker.GetChannel() wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { select { case <-channel: numEvents++ case <-quitChan: return } } }() time.Sleep(50 * time.Millisecond) ticker.Stop() time.Sleep(50 * time.Millisecond) close(quitChan) wg.Wait() assert.Equalf(t, 2, numEvents, "Expected 2 events, not %v", numEvents) } func TestPausableTicker_Restart(t *testing.T) { ticker := NewPausableTicker(20 * time.Millisecond) ticker.Start() numEvents := 0 quitChan := make(chan struct{}) channel := ticker.GetChannel() wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() for { select { case <-channel: numEvents++ case <-quitChan: return } } }() time.Sleep(50 * time.Millisecond) ticker.Stop() time.Sleep(50 * time.Millisecond) ticker.Start() time.Sleep(50 * time.Millisecond) ticker.Stop() close(quitChan) wg.Wait() assert.Equalf(t, 4, numEvents, "Expected 4 events, not %v", numEvents) } burrow-1.2.1/core/internal/helpers/validation.go000066400000000000000000000111531343357346000217330ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "net" "net/url" "regexp" "strconv" "strings" ) // ValidateIP returns true if the provided string can be parsed as an IP address (either IPv4 or IPv6). func ValidateIP(ipaddr string) bool { addr := net.ParseIP(ipaddr) return addr != nil } // ValidateHostname returns true if the provided string can be parsed as a hostname. In general this means: // // * One or more segments delimited by a '.' // * Each segment can be no more than 63 characters long // * Valid characters in a segment are letters, numbers, and dashes // * Segments may not start or end with a dash // * The exception is IPv6 addresses, which are also permitted. func ValidateHostname(hostname string) bool { matches, _ := regexp.MatchString(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`, hostname) if !matches { // Try as an IP address return ValidateIP(hostname) } return matches } // ValidateZookeeperPath returns true if the provided string can be parsed as a Zookeeper node path. This means that it // starts with a forward slash, and contains one or more segments that are separated by slashes (but does not end with // a slash). func ValidateZookeeperPath(path string) bool { parts := strings.Split(path, "/") if (len(parts) < 2) || (parts[0] != "") { return false } if (len(parts) == 2) && (parts[1] == "") { // Root node is OK return true } nodeRegexp, _ := regexp.Compile(`^[a-zA-Z0-9_\-][a-zA-Z0-9_\-.]*$`) for i, node := range parts { if i == 0 { continue } if !nodeRegexp.MatchString(node) { return false } } return true } // ValidateTopic returns true if the provided string is a valid topic name, which may only contain letters, numbers, // underscores, dashes, and periods. func ValidateTopic(topic string) bool { matches, _ := regexp.MatchString(`^[a-zA-Z0-9_.-]+$`, topic) return matches } // ValidateFilename returns true if the provided string is a sane-looking filename (not just a valid filename, which // could be almost anything). Right now, this is defined to be the same thing as ValidateTopic. func ValidateFilename(filename string) bool { return ValidateTopic(filename) } // ValidateEmail returns true if the provided string is an email address. This is a very simplistic validator - the // string must be of the form (something)@(something).(something) func ValidateEmail(email string) bool { matches, _ := regexp.MatchString(`^.+@.+\..+$`, email) return matches } // ValidateURL returns true if the provided string can be parsed as a URL. We use the net/url Parse func for this. func ValidateURL(rawURL string) bool { _, err := url.Parse(rawURL) return err == nil } // ValidateHostList returns true if the provided slice of strings can all be parsed by ValidateHostPort func ValidateHostList(hosts []string) bool { for _, host := range hosts { if !ValidateHostPort(host, false) { return false } } return true } // ValidateHostPort returns true if the provided string is of the form "hostname:port", where hostname is a valid // hostname or IP address (as parsed by ValidateIP or ValidateHostname), and port is a valid integer. func ValidateHostPort(host string, allowBlankHost bool) bool { // Must be hostname:port, ipv4:port, or [ipv6]:port. Optionally allow blank hostname hostname, portString, err := net.SplitHostPort(host) if err != nil { return false } // Validate the port is a numeric (yeah, strings are valid in some places, but we don't support it) _, err = strconv.Atoi(portString) if err != nil { return false } // Listeners can have blank hostnames, so we'll skip validation if that's what we're looking for if allowBlankHost && hostname == "" { return true } // Only IPv6 can contain : if strings.Contains(hostname, ":") && (!ValidateIP(hostname)) { return false } // If all the parts of the hostname are numbers, validate as IP. Otherwise, it's a hostname hostnameParts := strings.Split(hostname, ".") isIP4 := true for _, section := range hostnameParts { _, err := strconv.Atoi(section) if err != nil { isIP4 = false break } } if isIP4 { return ValidateIP(hostname) } return ValidateHostname(hostname) } burrow-1.2.1/core/internal/helpers/validation_test.go000066400000000000000000000137311343357346000227760ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "github.com/stretchr/testify/assert" "testing" ) type TestSet struct { TestValue string Result bool } var testIP = []TestSet{ {"1.2.3.4", true}, {"127.0.0.1", true}, {"204.27.175.1", true}, {"255.255.255.255", true}, {"256.1.2.3", false}, {"1.2.3.4.5", false}, {"notanip", false}, {"2001:0db8:0a0b:12f0:0000:0000:0000:0001", true}, {"2001:db8:a0b:12f0::1", true}, {"2001:db8::1", true}, {"2001:db8::2:1", true}, {"2001:db8:0:1:1:1:1:1", true}, {"2001:0db8:0a0b:12f0:0000:0000:0000:0001:0004", false}, } func TestValidateIP(t *testing.T) { for i, testSet := range testIP { result := ValidateIP(testSet.TestValue) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } var testHostnames = []TestSet{ {"hostname", true}, {"host0", true}, {"host.example.com", true}, {"example.com", true}, {"thissegmentiswaytoolongbecauseitshouldnotbemorethansixtythreecharacters.foo.com", false}, {"underscores_are.not.valid.com", false}, {"800.hostnames.starting.with.numbers.are.valid.because.people.suck.org", true}, {"hostnames-.may.not.end.with.a.dash.com", false}, {"no spaces.com", false}, } func TestValidateHostname(t *testing.T) { for i, testSet := range testHostnames { result := ValidateHostname(testSet.TestValue) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } var testZkPaths = []TestSet{ {"/", true}, {"", false}, {"/no/trailing/slash/", false}, {"/this/is/fine", true}, {"/underscores_are/ok", true}, {"/dashes-are/fine/too", true}, {"/no spaces/in/paths", false}, } func TestValidateZookeeperPath(t *testing.T) { for i, testSet := range testZkPaths { result := ValidateZookeeperPath(testSet.TestValue) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } var testTopics = []TestSet{ {"metrics", true}, {"__consumer_offsets", true}, {"stars*arent_valid_you_monster", false}, {"dashes-are-ok", true}, {"numbers0-are_fine", true}, {"no spaces", false}, {"dots.are_ok", true}, } func TestValidateTopic(t *testing.T) { for i, testSet := range testTopics { result := ValidateTopic(testSet.TestValue) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } var testEmails = []TestSet{ {"ok@example.com", true}, {"need@domain", false}, {"gotta.have.an.at", false}, {"nogood@", false}, {"this.is@ok.com", true}, } func TestValidateEmail(t *testing.T) { for i, testSet := range testEmails { result := ValidateEmail(testSet.TestValue) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } var testUrls = []TestSet{ {"http://foo.com/blah_blah", true}, {"http://foo.com/blah_blah/", true}, {"http://www.example.com/wpstyle/?p=364", true}, {"https://www.example.com/foo/?bar=baz&inga=42&quux", true}, {"http://✪df.ws/123", true}, {"http://userid:password@example.com:8080", true}, {"http://userid:password@example.com:8080/", true}, {"http://userid@example.com", true}, {"http://userid@example.com/", true}, {"http://userid@example.com:8080", true}, {"http://userid@example.com:8080/", true}, {"http://userid:password@example.com", true}, {"http://userid:password@example.com/", true}, {"http://142.42.1.1/", true}, {"http://142.42.1.1:8080/", true}, {"http://➡.ws/䨹", true}, {"http://⌘.ws", true}, {"http://⌘.ws/", true}, {"http://foo.com/blah_(wikipedia)#cite-1", true}, {"http://foo.com/blah_(wikipedia)_blah#cite-1", true}, {"http://foo.com/unicode_(✪)_in_parens", true}, {"http://foo.com/(something)?after=parens", true}, {"http://☺.damowmow.com/", true}, {"http://code.google.com/events/#&product=browser", true}, {"http://j.mp", true}, {"ftp://foo.bar/baz", true}, {"http://foo.bar/?q=Test%20URL-encoded%20stuff", true}, {"http://مثال.إختبار", true}, {"http://例子.测试", true}, {"http://उदाहरण.परीक्षा", true}, {"http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", true}, {"http://1337.net", true}, {"http://a.b-c.de", true}, {"http://223.255.255.254", true}, {"http:// shouldfail.com", false}, {":// should fail", false}, } func TestValidateUrl(t *testing.T) { for i, testSet := range testUrls { result := ValidateURL(testSet.TestValue) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } var testHostPorts = []TestSet{ {"1.2.3.4:3453", true}, {"127.0.0.1:2342", true}, {"204.27.175.1:4", true}, {"256.1.2.3:3743", false}, {"1.2.3.4.5:2452", false}, {"[2001:0db8:0a0b:12f0:0000:0000:0000:0001]:4356", true}, {"[2001:db8:a0b:12f0::1]:234", true}, {"[2001:db8::1]:3453", true}, {"2001:db8:0:1:1:1:1:1:3453", false}, {"[2001:0db8:0a0b:12f0:0000:0000:0000:0001:0004]:4533", false}, {"hostname:3432", true}, {"host0:4234", true}, {"host.example.com:23", true}, {"thissegmentiswaytoolongbecauseitshouldnotbemorethansixtythreecharacters.foo.com:36334", false}, {"underscores_are.not.valid.com:3453", false}, } func TestValidateHostList(t *testing.T) { for i, testSet := range testHostPorts { result := ValidateHostList([]string{testSet.TestValue}) assert.Equalf(t, testSet.Result, result, "Test %v - Expected '%v' to return %v, not %v", i, testSet.TestValue, testSet.Result, result) } } burrow-1.2.1/core/internal/helpers/zookeeper.go000066400000000000000000000150661343357346000216130ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "time" "github.com/linkedin/Burrow/core/protocol" "github.com/samuel/go-zookeeper/zk" "github.com/stretchr/testify/mock" "go.uber.org/zap" ) // BurrowZookeeperClient is an implementation of protocol.ZookeeperClient type BurrowZookeeperClient struct { client *zk.Conn } // ZookeeperConnect establishes a new connection to a pool of Zookeeper servers. The provided session timeout sets the // amount of time for which a session is considered valid after losing connection to a server. Within the session // timeout it's possible to reestablish a connection to a different server and keep the same session. This is means any // ephemeral nodes and watches are maintained. func ZookeeperConnect(servers []string, sessionTimeout time.Duration, logger *zap.Logger) (protocol.ZookeeperClient, <-chan zk.Event, error) { // We need a function to set the logger for the ZK connection zkSetLogger := func(c *zk.Conn) { c.SetLogger(zap.NewStdLog(logger)) } zkconn, connEventChan, err := zk.Connect(servers, sessionTimeout, zkSetLogger) return &BurrowZookeeperClient{client: zkconn}, connEventChan, err } // Close shuts down the connection to the Zookeeper ensemble. func (z *BurrowZookeeperClient) Close() { z.client.Close() } // ChildrenW returns a slice of names of child ZNodes immediately underneath the specified parent path. It also returns // a zk.Stat describing the parent path, and a channel over which a zk.Event object will be sent if the child list // changes (a child is added or deleted). func (z *BurrowZookeeperClient) ChildrenW(path string) ([]string, *zk.Stat, <-chan zk.Event, error) { return z.client.ChildrenW(path) } // GetW returns the data in the specified ZNode as a slice of bytes. It also returns a zk.Stat describing the ZNode, and // a channel over which a zk.Event object will be sent if the ZNode changes (data changed, or ZNode deleted). func (z *BurrowZookeeperClient) GetW(path string) ([]byte, *zk.Stat, <-chan zk.Event, error) { return z.client.GetW(path) } // ExistsW returns a boolean stating whether or not the specified path exists. This method also sets a watch on the node // (exists if it does not currently exist, or a data watch otherwise), providing an event channel that will receive a // message when the watch fires func (z *BurrowZookeeperClient) ExistsW(path string) (bool, *zk.Stat, <-chan zk.Event, error) { return z.client.ExistsW(path) } // Create makes a new ZNode at the specified path with the contents set to the data byte-slice. Flags can be provided // to specify that this is an ephemeral or sequence node, and an ACL must be provided. If no ACL is desired, specify // zk.WorldACL(zk.PermAll) func (z *BurrowZookeeperClient) Create(path string, data []byte, flags int32, acl []zk.ACL) (string, error) { return z.client.Create(path, data, flags, acl) } // NewLock creates a lock using the provided path. Multiple Zookeeper clients, using the same lock path, can synchronize // with each other to assure that only one client has the lock at any point. func (z *BurrowZookeeperClient) NewLock(path string) protocol.ZookeeperLock { return zk.NewLock(z.client, path, zk.WorldACL(zk.PermAll)) } // MockZookeeperClient is a mock of the protocol.ZookeeperClient interface to be used for testing. It should not be // used in normal code. type MockZookeeperClient struct { mock.Mock // InitialError can be set before using the MockZookeeperConnect call to specify an error that should be returned // from that call. InitialError error // EventChannel can be set before using the MockZookeeperConnect call to provide the channel that that call returns. EventChannel chan zk.Event // Servers stores the slice of strings that is provided to MockZookeeperConnect Servers []string // SessionTimeout stores the value that is provided to MockZookeeperConnect SessionTimeout time.Duration } // Close mocks protocol.ZookeeperClient.Close func (m *MockZookeeperClient) Close() { m.Called() if m.EventChannel != nil { close(m.EventChannel) } } // ChildrenW mocks protocol.ZookeeperClient.ChildrenW func (m *MockZookeeperClient) ChildrenW(path string) ([]string, *zk.Stat, <-chan zk.Event, error) { args := m.Called(path) return args.Get(0).([]string), args.Get(1).(*zk.Stat), args.Get(2).(<-chan zk.Event), args.Error(3) } // GetW mocks protocol.ZookeeperClient.GetW func (m *MockZookeeperClient) GetW(path string) ([]byte, *zk.Stat, <-chan zk.Event, error) { args := m.Called(path) return args.Get(0).([]byte), args.Get(1).(*zk.Stat), args.Get(2).(<-chan zk.Event), args.Error(3) } // ExistsW mocks protocol.ZookeeperClient.ExistsW func (m *MockZookeeperClient) ExistsW(path string) (bool, *zk.Stat, <-chan zk.Event, error) { args := m.Called(path) return args.Bool(0), args.Get(1).(*zk.Stat), args.Get(2).(<-chan zk.Event), args.Error(3) } // Create mocks protocol.ZookeeperClient.Create func (m *MockZookeeperClient) Create(path string, data []byte, flags int32, acl []zk.ACL) (string, error) { args := m.Called(path, data, flags, acl) return args.String(0), args.Error(1) } // NewLock mocks protocol.ZookeeperClient.NewLock func (m *MockZookeeperClient) NewLock(path string) protocol.ZookeeperLock { args := m.Called(path) return args.Get(0).(protocol.ZookeeperLock) } // MockZookeeperConnect is a func that mocks the ZookeeperConnect call, but allows us to pre-populate the return // values and save the arguments provided for assertions. func (m *MockZookeeperClient) MockZookeeperConnect(servers []string, sessionTimeout time.Duration, logger *zap.Logger) (protocol.ZookeeperClient, <-chan zk.Event, error) { m.Servers = servers m.SessionTimeout = sessionTimeout if m.EventChannel == nil { m.EventChannel = make(chan zk.Event) } return m, m.EventChannel, m.InitialError } // MockZookeeperLock is a mock of the protocol.ZookeeperLock interface. It should not be used in normal code. type MockZookeeperLock struct { mock.Mock } // Lock mocks protocol.ZookeeperLock.Lock func (m *MockZookeeperLock) Lock() error { args := m.Called() return args.Error(0) } // Unlock mocks protocol.ZookeeperLock.Unlock func (m *MockZookeeperLock) Unlock() error { args := m.Called() return args.Error(0) } burrow-1.2.1/core/internal/helpers/zookeeper_test.go000066400000000000000000000016231343357346000226440ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package helpers import ( "github.com/linkedin/Burrow/core/protocol" "github.com/stretchr/testify/assert" "testing" ) func TestBurrowZookeeperClient_ImplementsZookeeperClient(t *testing.T) { assert.Implements(t, (*protocol.ZookeeperClient)(nil), new(BurrowZookeeperClient)) } func TestMockZookeeperClient_ImplementsZookeeperClient(t *testing.T) { assert.Implements(t, (*protocol.ZookeeperClient)(nil), new(MockZookeeperClient)) } burrow-1.2.1/core/internal/httpserver/000077500000000000000000000000001343357346000200155ustar00rootroot00000000000000burrow-1.2.1/core/internal/httpserver/config.go000066400000000000000000000277751343357346000216330ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package httpserver import ( "net/http" "github.com/julienschmidt/httprouter" "github.com/spf13/viper" ) func (hc *Coordinator) configMain(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { // Build JSON structs for config configGeneral := httpResponseConfigGeneral{ PIDFile: viper.GetString("general.pidfile"), StdoutLogfile: viper.GetString("general.stdout-logfile"), AccessControlAllowOrigin: viper.GetString("general.access-control-allow-origin"), } configLogging := httpResponseConfigLogging{ Filename: viper.GetString("logging.filename"), MaxSize: viper.GetInt("logging.maxsize"), MaxBackups: viper.GetInt("logging.maxbackups"), MaxAge: viper.GetInt("logging.maxage"), UseLocalTime: viper.GetBool("logging.use-localtime"), UseCompression: viper.GetBool("logging.use-compression"), Level: viper.GetString("logging.level"), } configZookeeper := httpResponseConfigZookeeper{ Servers: viper.GetStringSlice("zookeeper.servers"), Timeout: viper.GetInt("zookeeper.timeout"), RootPath: viper.GetString("zookeeper.root-path"), } servers := viper.GetStringMap("httpserver") configHTTPServer := make(map[string]httpResponseConfigHTTPServer) for name := range servers { configRoot := "httpserver." + name configHTTPServer[name] = httpResponseConfigHTTPServer{ Address: viper.GetString(configRoot + ".address"), Timeout: viper.GetInt(configRoot + ".timeout"), TLS: viper.GetString(configRoot + ".tls"), } } requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigMain{ Error: false, Message: "main config returned", General: configGeneral, Logging: configLogging, Zookeeper: configZookeeper, HTTPServer: configHTTPServer, Request: requestInfo, }) } func (hc *Coordinator) writeModuleListResponse(w http.ResponseWriter, r *http.Request, coordinator string, modules []string) { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleList{ Error: false, Message: "module list returned", Request: requestInfo, Coordinator: coordinator, Modules: modules, }) } func (hc *Coordinator) configStorageList(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { modules := viper.GetStringMap("storage") moduleList := make([]string, len(modules)) i := 0 for name := range modules { moduleList[i] = name i++ } hc.writeModuleListResponse(w, r, "storage", moduleList) } func (hc *Coordinator) configConsumerList(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { modules := viper.GetStringMap("consumer") moduleList := make([]string, len(modules)) i := 0 for name := range modules { moduleList[i] = name i++ } hc.writeModuleListResponse(w, r, "consumer", moduleList) } func (hc *Coordinator) configClusterList(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { modules := viper.GetStringMap("cluster") moduleList := make([]string, len(modules)) i := 0 for name := range modules { moduleList[i] = name i++ } hc.writeModuleListResponse(w, r, "cluster", moduleList) } func (hc *Coordinator) configEvaluatorList(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { modules := viper.GetStringMap("evaluator") moduleList := make([]string, len(modules)) i := 0 for name := range modules { moduleList[i] = name i++ } hc.writeModuleListResponse(w, r, "evaluator", moduleList) } func (hc *Coordinator) configNotifierList(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { modules := viper.GetStringMap("notifier") moduleList := make([]string, len(modules)) i := 0 for name := range modules { moduleList[i] = name i++ } hc.writeModuleListResponse(w, r, "notifier", moduleList) } func (hc *Coordinator) configStorageDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { configRoot := "storage." + params.ByName("name") if !viper.IsSet(configRoot) { hc.writeErrorResponse(w, r, http.StatusNotFound, "storage module not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "storage module detail returned", Module: httpResponseConfigModuleStorage{ ClassName: viper.GetString(configRoot + ".class-name"), Intervals: viper.GetInt(configRoot + ".intervals"), MinDistance: viper.GetInt64(configRoot + ".min-distance"), GroupWhitelist: viper.GetString(configRoot + ".group-whitelist"), ExpireGroup: viper.GetInt64(configRoot + ".expire-group"), }, Request: requestInfo, }) } } func (hc *Coordinator) configConsumerDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { configRoot := "consumer." + params.ByName("name") if !viper.IsSet(configRoot) { hc.writeErrorResponse(w, r, http.StatusNotFound, "consumer module not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "consumer module detail returned", Module: httpResponseConfigModuleConsumer{ ClassName: viper.GetString(configRoot + ".class-name"), Cluster: viper.GetString(configRoot + ".cluster"), Servers: viper.GetStringSlice(configRoot + ".servers"), GroupWhitelist: viper.GetString(configRoot + ".group-whitelist"), ZookeeperPath: viper.GetString(configRoot + ".zookeeper-path"), ZookeeperTimeout: int32(viper.GetInt64(configRoot + ".zookeeper-timeout")), ClientProfile: getClientProfile(viper.GetString(configRoot + ".client-profile")), OffsetsTopic: viper.GetString(configRoot + ".offsets-topic"), StartLatest: viper.GetBool(configRoot + ".start-latest"), }, Request: requestInfo, }) } } func (hc *Coordinator) configEvaluatorDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { configRoot := "evaluator." + params.ByName("name") if !viper.IsSet(configRoot) { hc.writeErrorResponse(w, r, http.StatusNotFound, "evaluator module not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "evaluator module detail returned", Module: httpResponseConfigModuleEvaluator{ ClassName: viper.GetString(configRoot + ".class-name"), ExpireCache: viper.GetInt64(configRoot + ".expire-cache"), }, Request: requestInfo, }) } } func (hc *Coordinator) configNotifierHTTP(w http.ResponseWriter, r *http.Request, configRoot string) { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "notifier module detail returned", Module: httpResponseConfigModuleNotifierHTTP{ ClassName: viper.GetString(configRoot + ".class-name"), GroupWhitelist: viper.GetString(configRoot + ".group-whitelist"), Interval: viper.GetInt64(configRoot + ".interval"), Threshold: viper.GetInt(configRoot + ".threshold"), Timeout: viper.GetInt(configRoot + ".timeout"), Keepalive: viper.GetInt(configRoot + ".keepalive"), URLOpen: viper.GetString(configRoot + ".url-open"), URLClose: viper.GetString(configRoot + ".url-close"), MethodOpen: viper.GetString(configRoot + ".method-open"), MethodClose: viper.GetString(configRoot + ".method-close"), TemplateOpen: viper.GetString(configRoot + ".template-open"), TemplateClose: viper.GetString(configRoot + ".template-close"), Extras: viper.GetStringMapString(configRoot + ".extras"), SendClose: viper.GetBool(configRoot + ".send-close"), }, Request: requestInfo, }) } func (hc *Coordinator) configNotifierSlack(w http.ResponseWriter, r *http.Request, configRoot string) { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "notifier module detail returned", Module: httpResponseConfigModuleNotifierSlack{ ClassName: viper.GetString(configRoot + ".class-name"), GroupWhitelist: viper.GetString(configRoot + ".group-whitelist"), Interval: viper.GetInt64(configRoot + ".interval"), Threshold: viper.GetInt(configRoot + ".threshold"), Timeout: viper.GetInt(configRoot + ".timeout"), Keepalive: viper.GetInt(configRoot + ".keepalive"), TemplateOpen: viper.GetString(configRoot + ".template-open"), TemplateClose: viper.GetString(configRoot + ".template-close"), Extras: viper.GetStringMapString(configRoot + ".extras"), SendClose: viper.GetBool(configRoot + ".send-close"), Channel: viper.GetString(configRoot + ".channel"), Username: viper.GetString(configRoot + ".username"), IconURL: viper.GetString(configRoot + ".icon-url"), IconEmoji: viper.GetString(configRoot + ".icon-emoji"), }, Request: requestInfo, }) } func (hc *Coordinator) configNotifierEmail(w http.ResponseWriter, r *http.Request, configRoot string) { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "notifier module detail returned", Module: httpResponseConfigModuleNotifierEmail{ ClassName: viper.GetString(configRoot + ".class-name"), GroupWhitelist: viper.GetString(configRoot + ".group-whitelist"), Interval: viper.GetInt64(configRoot + ".interval"), Threshold: viper.GetInt(configRoot + ".threshold"), TemplateOpen: viper.GetString(configRoot + ".template-open"), TemplateClose: viper.GetString(configRoot + ".template-close"), Extras: viper.GetStringMapString(configRoot + ".extras"), SendClose: viper.GetBool(configRoot + ".send-close"), Server: viper.GetString(configRoot + ".server"), Port: viper.GetInt(configRoot + ".port"), AuthType: viper.GetString(configRoot + ".auth-type"), Username: viper.GetString(configRoot + ".username"), From: viper.GetString(configRoot + ".from"), To: viper.GetString(configRoot + ".to"), }, Request: requestInfo, }) } func (hc *Coordinator) configNotifierNull(w http.ResponseWriter, r *http.Request, configRoot string) { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "notifier module detail returned", Module: httpResponseConfigModuleNotifierNull{ ClassName: viper.GetString(configRoot + ".class-name"), GroupWhitelist: viper.GetString(configRoot + ".group-whitelist"), Interval: viper.GetInt64(configRoot + ".interval"), Threshold: viper.GetInt(configRoot + ".threshold"), TemplateOpen: viper.GetString(configRoot + ".template-open"), TemplateClose: viper.GetString(configRoot + ".template-close"), Extras: viper.GetStringMapString(configRoot + ".extras"), SendClose: viper.GetBool(configRoot + ".send-close"), }, Request: requestInfo, }) } func (hc *Coordinator) configNotifierDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { configRoot := "notifier." + params.ByName("name") if !viper.IsSet(configRoot) { hc.writeErrorResponse(w, r, http.StatusNotFound, "notifier module not found") } else { // Return the right profile structure switch viper.GetString(configRoot + ".class-name") { case "http": hc.configNotifierHTTP(w, r, configRoot) case "email": hc.configNotifierEmail(w, r, configRoot) case "slack": hc.configNotifierSlack(w, r, configRoot) case "null": hc.configNotifierNull(w, r, configRoot) } } } burrow-1.2.1/core/internal/httpserver/config_test.go000066400000000000000000000314461343357346000226600ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package httpserver import ( "encoding/json" "net/http" "github.com/stretchr/testify/assert" "net/http/httptest" "testing" "github.com/spf13/viper" ) func setupConfiguration() { viper.Reset() viper.Set("client-profile.test.client-id", "testid") viper.Set("storage.teststorage.class-name", "inmemory") viper.Set("consumer.testconsumer.class-name", "kafka_zk") viper.Set("consumer.testconsumer.client-profile", "test") viper.Set("cluster.testcluster.class-name", "kafka") viper.Set("cluster.testcluster.client-profile", "test") viper.Set("evaluator.testevaluator.class-name", "caching") viper.Set("notifier.testnotifier.class-name", "null") } func TestHttpServer_configMain(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body - just test that it decodes decoder := json.NewDecoder(rr.Body) var resp httpResponseConfigMain err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") } func TestHttpServer_configStorageList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/storage", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConfigModuleList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "storage", resp.Coordinator, "Expected Coordinator to be storage, not %v", resp.Coordinator) assert.Equalf(t, []string{"teststorage"}, resp.Modules, "Expected Modules to be [teststorage], not %v", resp.Modules) } func TestHttpServer_configConsumerList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/consumer", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConfigModuleList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "consumer", resp.Coordinator, "Expected Coordinator to be consumer, not %v", resp.Coordinator) assert.Equalf(t, []string{"testconsumer"}, resp.Modules, "Expected Modules to be [testconsumer], not %v", resp.Modules) } func TestHttpServer_configClusterList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/cluster", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConfigModuleList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "cluster", resp.Coordinator, "Expected Coordinator to be cluster, not %v", resp.Coordinator) assert.Equalf(t, []string{"testcluster"}, resp.Modules, "Expected Modules to be [testcluster], not %v", resp.Modules) } func TestHttpServer_configEvaluatorList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/evaluator", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConfigModuleList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "evaluator", resp.Coordinator, "Expected Coordinator to be evaluator, not %v", resp.Coordinator) assert.Equalf(t, []string{"testevaluator"}, resp.Modules, "Expected Modules to be [testevaluator], not %v", resp.Modules) } func TestHttpServer_configNotifierList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/notifier", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConfigModuleList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "notifier", resp.Coordinator, "Expected Coordinator to be notifier, not %v", resp.Coordinator) assert.Equalf(t, []string{"testnotifier"}, resp.Modules, "Expected Modules to be [testnotifier], not %v", resp.Modules) } func TestHttpServer_configStorageDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/storage/teststorage", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Need a custom type for the test, due to variations in the response for different module types type ResponseType struct { Error bool `json:"error"` Message string `json:"message"` Module httpResponseConfigModuleStorage `json:"module"` Request httpResponseRequestInfo `json:"request"` } // Parse response body decoder := json.NewDecoder(rr.Body) var resp ResponseType err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "inmemory", resp.Module.ClassName, "Expected ClassName to be immemory, not %v", resp.Module.ClassName) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/config/storage/nomodule", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_configConsumerDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/consumer/testconsumer", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Need a custom type for the test, due to variations in the response for different module types type ResponseType struct { Error bool `json:"error"` Message string `json:"message"` Module httpResponseConfigModuleConsumer `json:"module"` Request httpResponseRequestInfo `json:"request"` } // Parse response body decoder := json.NewDecoder(rr.Body) var resp ResponseType err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "kafka_zk", resp.Module.ClassName, "Expected ClassName to be kafka_zk, not %v", resp.Module.ClassName) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/config/consumer/nomodule", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_configEvaluatorDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/evaluator/testevaluator", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Need a custom type for the test, due to variations in the response for different module types type ResponseType struct { Error bool `json:"error"` Message string `json:"message"` Module httpResponseConfigModuleEvaluator `json:"module"` Request httpResponseRequestInfo `json:"request"` } // Parse response body decoder := json.NewDecoder(rr.Body) var resp ResponseType err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "caching", resp.Module.ClassName, "Expected ClassName to be caching, not %v", resp.Module.ClassName) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/config/evaluator/nomodule", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_configNotifierDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() setupConfiguration() // Set up a request req, err := http.NewRequest("GET", "/v3/config/notifier/testnotifier", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Need a custom type for the test, due to variations in the response for different module types type ResponseType struct { Error bool `json:"error"` Message string `json:"message"` Module httpResponseConfigModuleNotifierNull `json:"module"` Request httpResponseRequestInfo `json:"request"` } // Parse response body decoder := json.NewDecoder(rr.Body) var resp ResponseType err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "null", resp.Module.ClassName, "Expected ClassName to be null, not %v", resp.Module.ClassName) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/config/notifier/nomodule", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } burrow-1.2.1/core/internal/httpserver/coordinator.go000066400000000000000000000267271343357346000227050ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package httpserver - HTTP API endpoint // The httpserver subsystem provides an HTTP interface to Burrow that can be used to fetch information about the // clusters and consumers it is monitoring. More documentation on the requests and responses is provided at // https://github.com/linkedin/Burrow/wiki/HTTP-Endpoint. package httpserver import ( "crypto/tls" "crypto/x509" "encoding/json" "errors" "io/ioutil" "net" "net/http" "os" "strings" "time" "github.com/julienschmidt/httprouter" "github.com/spf13/viper" "go.uber.org/zap" "go.uber.org/zap/zapcore" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // Coordinator runs the HTTP interface for Burrow, managing all configured listeners. type Coordinator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger router *httprouter.Router servers map[string]*http.Server } // Configure is called to configure the HTTP server. This includes validating all configurations for each configured // listener (which are not treated as separate modules, as opposed to other coordinators), as well as setting up the // request router. Any configuration failure will cause the func to panic with an appropriate error message. // // If no listener has been configured, the coordinator will set up a default listener on a random port greater than // 1024, as selected by the net.Listener call. This listener will be logged so that the port chosen will be known. func (hc *Coordinator) Configure() { hc.Log.Info("configuring") hc.router = httprouter.New() // If no HTTP server configured, add a default HTTP server that listens on a random port servers := viper.GetStringMap("httpserver") if len(servers) == 0 { viper.Set("httpserver.default.address", ":0") servers = viper.GetStringMap("httpserver") } // Validate provided HTTP server configs hc.servers = make(map[string]*http.Server) for name := range servers { configRoot := "httpserver." + name server := &http.Server{ Handler: hc.router, } server.Addr = viper.GetString(configRoot + ".address") if !helpers.ValidateHostPort(server.Addr, true) { panic("invalid HTTP server listener address") } viper.SetDefault(configRoot+".timeout", 300) timeout := viper.GetInt(configRoot + ".timeout") server.ReadTimeout = time.Duration(timeout) * time.Second server.ReadHeaderTimeout = time.Duration(timeout) * time.Second server.WriteTimeout = time.Duration(timeout) * time.Second server.IdleTimeout = time.Duration(timeout) * time.Second if viper.IsSet(configRoot + ".tls") { tlsName := viper.GetString(configRoot + ".tls") certFile := viper.GetString("tls." + tlsName + ".certfile") keyFile := viper.GetString("tls." + tlsName + ".keyfile") caFile := viper.GetString("tls." + tlsName + ".cafile") server.TLSConfig = &tls.Config{} if caFile != "" { caCert, err := ioutil.ReadFile(caFile) if err != nil { panic("cannot read TLS CA file: " + err.Error()) } server.TLSConfig.RootCAs = x509.NewCertPool() server.TLSConfig.RootCAs.AppendCertsFromPEM(caCert) } if certFile == "" || keyFile == "" { panic("TLS HTTP server specified with missing certificate or key") } cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { panic("cannot read TLS certificate or key file: " + err.Error()) } server.TLSConfig.Certificates = []tls.Certificate{cert} server.TLSConfig.BuildNameToCertificate() } hc.servers[name] = server } // Configure URL routes here // This is a catchall for undefined URLs hc.router.NotFound = &defaultHandler{} // This is a healthcheck URL. Please don't change it hc.router.GET("/burrow/admin", hc.handleAdmin) // All valid paths go here hc.router.GET("/v3/kafka", hc.handleClusterList) hc.router.GET("/v3/kafka/:cluster", hc.handleClusterDetail) hc.router.GET("/v3/kafka/:cluster/topic", hc.handleTopicList) hc.router.GET("/v3/kafka/:cluster/topic/:topic", hc.handleTopicDetail) hc.router.GET("/v3/kafka/:cluster/topic/:topic/consumers", hc.handleTopicConsumerList) hc.router.GET("/v3/kafka/:cluster/consumer", hc.handleConsumerList) hc.router.GET("/v3/kafka/:cluster/consumer/:consumer", hc.handleConsumerDetail) hc.router.GET("/v3/kafka/:cluster/consumer/:consumer/status", hc.handleConsumerStatus) hc.router.GET("/v3/kafka/:cluster/consumer/:consumer/lag", hc.handleConsumerStatusComplete) hc.router.GET("/v3/config", hc.configMain) hc.router.GET("/v3/config/storage", hc.configStorageList) hc.router.GET("/v3/config/storage/:name", hc.configStorageDetail) hc.router.GET("/v3/config/evaluator", hc.configEvaluatorList) hc.router.GET("/v3/config/evaluator/:name", hc.configEvaluatorDetail) hc.router.GET("/v3/config/cluster", hc.configClusterList) hc.router.GET("/v3/config/cluster/:cluster", hc.handleClusterDetail) hc.router.GET("/v3/config/consumer", hc.configConsumerList) hc.router.GET("/v3/config/consumer/:name", hc.configConsumerDetail) hc.router.GET("/v3/config/notifier", hc.configNotifierList) hc.router.GET("/v3/config/notifier/:name", hc.configNotifierDetail) // TODO: This should really have authentication protecting it hc.router.DELETE("/v3/kafka/:cluster/consumer/:consumer", hc.handleConsumerDelete) hc.router.GET("/v3/admin/loglevel", hc.getLogLevel) hc.router.POST("/v3/admin/loglevel", hc.setLogLevel) } // Start is responsible for starting the listener on each configured address. If any listener fails to start, the error // is logged, and the listeners that have already been started are stopped. The func then returns the error encountered // to the caller. Once the listeners are all started, the HTTP server itself is started on each listener to respond to // requests. func (hc *Coordinator) Start() error { hc.Log.Info("starting") // Start listeners listeners := make(map[string]net.Listener) for name, server := range hc.servers { ln, err := net.Listen("tcp", hc.servers[name].Addr) if err != nil { hc.Log.Error("failed to listen", zap.String("listener", hc.servers[name].Addr), zap.Error(err)) for _, listenerToClose := range listeners { if listenerToClose != nil { closeErr := listenerToClose.Close() if closeErr != nil { hc.Log.Error("could not close listener: %v", zap.Error(closeErr)) } } } return err } hc.Log.Info("started listener", zap.String("listener", ln.Addr().String())) listeners[name] = tcpKeepAliveListener{ Keepalive: server.IdleTimeout, TCPListener: ln.(*net.TCPListener), } } // Start the HTTP server on the listeners for name, server := range hc.servers { go server.Serve(listeners[name]) } return nil } // Stop calls the Close func for each configured HTTP server listener. This stops the underlying HTTP server without // waiting for client calls to complete. If there are any errors while shutting down the listeners, this does not stop // other listeners from being closed. A generic error will be returned to the caller in this case. func (hc *Coordinator) Stop() error { hc.Log.Info("shutdown") // Close all servers collectedErrors := make([]zapcore.Field, 0) for _, server := range hc.servers { err := server.Close() if err != nil { collectedErrors = append(collectedErrors, zap.Error(err)) } } if len(collectedErrors) > 0 { hc.Log.Error("errors shutting down", collectedErrors...) return errors.New("error shutting down HTTP servers") } return nil } // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted connections. It's used by ListenAndServe and // ListenAndServeTLS so dead TCP connections (e.g. closing laptop mid-download) eventually go away. type tcpKeepAliveListener struct { *net.TCPListener Keepalive time.Duration } func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { tc, err := ln.AcceptTCP() if err != nil { return } if ln.Keepalive > 0 { tc.SetKeepAlive(true) tc.SetKeepAlivePeriod(ln.Keepalive) } return tc, nil } func makeRequestInfo(r *http.Request) httpResponseRequestInfo { hostname, _ := os.Hostname() return httpResponseRequestInfo{ URI: r.URL.Path, Host: hostname, } } func (hc *Coordinator) writeResponse(w http.ResponseWriter, r *http.Request, statusCode int, jsonObj interface{}) { // Add CORS header, if configured corsHeader := viper.GetString("general.access-control-allow-origin") if corsHeader != "" { w.Header().Set("Access-Control-Allow-Origin", corsHeader) } w.Header().Set("Content-Type", "application/json") if jsonBytes, err := json.Marshal(jsonObj); err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("{\"error\":true,\"message\":\"could not encode JSON\",\"result\":{}}")) } else { w.WriteHeader(statusCode) w.Write(jsonBytes) } } func (hc *Coordinator) writeErrorResponse(w http.ResponseWriter, r *http.Request, errValue int, message string) { hc.writeResponse(w, r, errValue, httpResponseError{ Error: true, Message: message, Request: makeRequestInfo(r), }) } // This is a catch-all handler for unknown URLs. It should return a 404 type defaultHandler struct{} func (handler *defaultHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "{\"error\":true,\"message\":\"invalid request type\",\"result\":{}}", http.StatusNotFound) } func (hc *Coordinator) handleAdmin(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { // Add CORS header, if configured corsHeader := viper.GetString("general.access-control-allow-origin") if corsHeader != "" { w.Header().Set("Access-Control-Allow-Origin", corsHeader) } w.WriteHeader(http.StatusOK) w.Write([]byte("GOOD")) } func (hc *Coordinator) getLogLevel(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseLogLevel{ Error: false, Message: "log level returned", Level: hc.App.LogLevel.Level().String(), Request: requestInfo, }) } func (hc *Coordinator) setLogLevel(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { // Decode the JSON body decoder := json.NewDecoder(r.Body) var req logLevelRequest err := decoder.Decode(&req) if err != nil { hc.writeErrorResponse(w, r, http.StatusBadRequest, "could not decode message body") return } r.Body.Close() // Explicitly validate the log level provided switch strings.ToLower(req.Level) { case "debug", "trace": hc.App.LogLevel.SetLevel(zap.DebugLevel) case "info": hc.App.LogLevel.SetLevel(zap.InfoLevel) case "warning", "warn": hc.App.LogLevel.SetLevel(zap.WarnLevel) case "error": hc.App.LogLevel.SetLevel(zap.ErrorLevel) case "fatal": hc.App.LogLevel.SetLevel(zap.FatalLevel) default: hc.writeErrorResponse(w, r, http.StatusNotFound, "unknown log level") return } requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseError{ Error: false, Message: "set log level", Request: requestInfo, }) } burrow-1.2.1/core/internal/httpserver/coordinator_test.go000066400000000000000000000107701343357346000237330ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package httpserver import ( "encoding/json" "net/http" "strings" "time" "github.com/stretchr/testify/assert" "net/http/httptest" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) func fixtureConfiguredCoordinator() *Coordinator { logLevel := zap.NewAtomicLevelAt(zap.InfoLevel) coordinator := Coordinator{ Log: zap.NewNop(), App: &protocol.ApplicationContext{ Logger: zap.NewNop(), LogLevel: &logLevel, StorageChannel: make(chan *protocol.StorageRequest), EvaluatorChannel: make(chan *protocol.EvaluatorRequest), }, } viper.Reset() coordinator.Configure() return &coordinator } func TestHttpServer_handleAdmin(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Set up a request req, err := http.NewRequest("GET", "/burrow/admin", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) assert.Equalf(t, "GOOD", rr.Body.String(), "Expected response body to be 'GOOD', not '%v'", rr.Body.String()) } func TestHttpServer_getClusterList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchClusters, request.RequestType, "Expected request of type StorageFetchClusters, not %v", request.RequestType) request.Reply <- []string{"testcluster"} close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/admin/loglevel", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseLogLevel err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "info", resp.Level, "Expected Level to be info, not %v", resp.Level) } func TestHttpServer_setLogLevel(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Set up a request req, err := http.NewRequest("POST", "/v3/admin/loglevel", strings.NewReader("{\"level\": \"debug\"}")) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseError err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") // The log level is changed async to the HTTP call, so sleep to make sure it got processed time.Sleep(100 * time.Millisecond) assert.Equalf(t, zap.DebugLevel, coordinator.App.LogLevel.Level(), "Expected log level to be set to Debug, not %v", coordinator.App.LogLevel.Level().String()) } func TestHttpServer_DefaultHandler(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Set up a request req, err := http.NewRequest("GET", "/v3/no/such/uri", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseError err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.True(t, resp.Error, "Expected response Error to be true") } burrow-1.2.1/core/internal/httpserver/kafka.go000066400000000000000000000216221343357346000214240ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package httpserver import ( "net/http" "github.com/julienschmidt/httprouter" "github.com/spf13/viper" "github.com/linkedin/Burrow/core/protocol" ) func (hc *Coordinator) handleClusterList(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { // Fetch cluster list from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchClusters, Reply: make(chan interface{}), } hc.App.StorageChannel <- request response := <-request.Reply requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseClusterList{ Error: false, Message: "cluster list returned", Clusters: response.([]string), Request: requestInfo, }) } func getTLSProfile(name string) *httpResponseTLSProfile { configRoot := "tls." + name if !viper.IsSet(configRoot) { return nil } return &httpResponseTLSProfile{ Name: name, CertFile: viper.GetString(configRoot + ".certfile"), KeyFile: viper.GetString(configRoot + ".keyfile"), CAFile: viper.GetString(configRoot + ".cafile"), NoVerify: viper.GetBool(configRoot + ".noverify"), } } func getSASLProfile(name string) *httpResponseSASLProfile { configRoot := "sasl." + name if !viper.IsSet(configRoot) { return nil } return &httpResponseSASLProfile{ Name: name, HandshakeFirst: viper.GetBool(configRoot + ".handshake-first"), Username: viper.GetString(configRoot + ".username"), } } func getClientProfile(name string) httpResponseClientProfile { configRoot := "client-profile." + name return httpResponseClientProfile{ Name: name, ClientID: viper.GetString(configRoot + ".client-id"), KafkaVersion: viper.GetString(configRoot + ".kafka-version"), TLS: getTLSProfile(viper.GetString(configRoot + ".tls")), SASL: getSASLProfile(viper.GetString(configRoot + ".sasl")), } } func (hc *Coordinator) handleClusterDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Get cluster config configRoot := "cluster." + params.ByName("cluster") if !viper.IsSet(configRoot) { hc.writeErrorResponse(w, r, http.StatusNotFound, "cluster module not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConfigModuleDetail{ Error: false, Message: "cluster module detail returned", Module: httpResponseConfigModuleCluster{ ClassName: viper.GetString(configRoot + ".class-name"), Servers: viper.GetStringSlice(configRoot + ".servers"), TopicRefresh: viper.GetInt64(configRoot + ".topic-refresh"), OffsetRefresh: viper.GetInt64(configRoot + ".offset-refresh"), ClientProfile: getClientProfile(viper.GetString(configRoot + ".client-profile")), }, Request: requestInfo, }) } } func (hc *Coordinator) handleTopicList(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch topic list from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchTopics, Cluster: params.ByName("cluster"), Reply: make(chan interface{}), } hc.App.StorageChannel <- request response := <-request.Reply if response == nil { hc.writeErrorResponse(w, r, http.StatusNotFound, "cluster not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseTopicList{ Error: false, Message: "topic list returned", Topics: response.([]string), Request: requestInfo, }) } } func (hc *Coordinator) handleTopicDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch topic offsets from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchTopic, Cluster: params.ByName("cluster"), Topic: params.ByName("topic"), Reply: make(chan interface{}), } hc.App.StorageChannel <- request response := <-request.Reply if response == nil { hc.writeErrorResponse(w, r, http.StatusNotFound, "cluster or topic not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseTopicDetail{ Error: false, Message: "topic offsets returned", Offsets: response.([]int64), Request: requestInfo, }) } } func (hc *Coordinator) handleTopicConsumerList(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch topic offsets from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumersForTopic, Cluster: params.ByName("cluster"), Topic: params.ByName("topic"), Reply: make(chan interface{}), } hc.App.StorageChannel <- request response := <-request.Reply if response == nil { hc.writeErrorResponse(w, r, http.StatusNotFound, "cluster not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseTopicConsumerDetail{ Error: false, Message: "consumers of topic returned", Consumers: response.([]string), Request: requestInfo, }) } } func (hc *Coordinator) handleConsumerList(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch consumer list from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumers, Cluster: params.ByName("cluster"), Reply: make(chan interface{}), } hc.App.StorageChannel <- request response := <-request.Reply if response == nil { hc.writeErrorResponse(w, r, http.StatusNotFound, "cluster not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConsumerList{ Error: false, Message: "consumer list returned", Consumers: response.([]string), Request: requestInfo, }) } } func (hc *Coordinator) handleConsumerDetail(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch consumer data from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumer, Cluster: params.ByName("cluster"), Group: params.ByName("consumer"), Reply: make(chan interface{}), } hc.App.StorageChannel <- request response := <-request.Reply if response == nil { hc.writeErrorResponse(w, r, http.StatusNotFound, "cluster or consumer not found") } else { requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseConsumerDetail{ Error: false, Message: "consumer detail returned", Topics: response.(protocol.ConsumerTopics), Request: requestInfo, }) } } func (hc *Coordinator) handleConsumerStatus(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch consumer data from the storage module request := &protocol.EvaluatorRequest{ Cluster: params.ByName("cluster"), Group: params.ByName("consumer"), ShowAll: false, Reply: make(chan *protocol.ConsumerGroupStatus), } hc.App.EvaluatorChannel <- request response := <-request.Reply responseCode := http.StatusOK if response.Status == protocol.StatusNotFound { responseCode = http.StatusNotFound } requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, responseCode, httpResponseConsumerStatus{ Error: false, Message: "consumer status returned", Status: *response, Request: requestInfo, }) } func (hc *Coordinator) handleConsumerStatusComplete(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Fetch consumer data from the storage module request := &protocol.EvaluatorRequest{ Cluster: params.ByName("cluster"), Group: params.ByName("consumer"), ShowAll: true, Reply: make(chan *protocol.ConsumerGroupStatus), } hc.App.EvaluatorChannel <- request response := <-request.Reply responseCode := http.StatusOK if response.Status == protocol.StatusNotFound { responseCode = http.StatusNotFound } requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, responseCode, httpResponseConsumerStatus{ Error: false, Message: "consumer status returned", Status: *response, Request: requestInfo, }) } func (hc *Coordinator) handleConsumerDelete(w http.ResponseWriter, r *http.Request, params httprouter.Params) { // Delete consumer from the storage module request := &protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteGroup, Cluster: params.ByName("cluster"), Group: params.ByName("consumer"), } hc.App.StorageChannel <- request requestInfo := makeRequestInfo(r) hc.writeResponse(w, r, http.StatusOK, httpResponseError{ Error: false, Message: "consumer group removed", Request: requestInfo, }) } burrow-1.2.1/core/internal/httpserver/kafka_test.go000066400000000000000000000620661343357346000224720ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package httpserver import ( "encoding/json" "net/http" "time" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "net/http/httptest" "testing" "github.com/linkedin/Burrow/core/protocol" ) func TestHttpServer_handleClusterList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchClusters, request.RequestType, "Expected request of type StorageFetchClusters, not %v", request.RequestType) request.Reply <- []string{"testcluster"} close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/kafka", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseClusterList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, []string{"testcluster"}, resp.Clusters, "Expected Clusters list to contain just testcluster, not %v", resp.Clusters) } func TestHttpServer_handleClusterDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() viper.Set("client-profile.test.client-id", "testid") viper.Set("cluster.testcluster.class-name", "kafka") viper.Set("cluster.testcluster.client-profile", "test") // Set up a request req, err := http.NewRequest("GET", "/v3/kafka/testcluster", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Need a specialized version of this for decoding type ResponseType struct { Error bool `json:"error"` Message string `json:"message"` Module httpResponseConfigModuleCluster `json:"module"` Request httpResponseRequestInfo `json:"request"` } // Parse response body decoder := json.NewDecoder(rr.Body) var resp ResponseType err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, "kafka", resp.Module.ClassName, "Expected response to contain a module with type kafka, not %v", resp.Module.ClassName) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_handleTopicList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchTopics, request.RequestType, "Expected request of type StorageFetchTopics, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) request.Reply <- []string{"testtopic"} close(request.Reply) // Second request is a 404 request = <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchTopics, request.RequestType, "Expected request of type StorageFetchTopics, not %v", request.RequestType) assert.Equalf(t, "nocluster", request.Cluster, "Expected request Cluster to be nocluster, not %v", request.Cluster) close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/kafka/testcluster/topic", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseTopicList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, []string{"testtopic"}, resp.Topics, "Expected Topics list to contain just testtopic, not %v", resp.Topics) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster/topic", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_handleConsumerList(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchConsumers, request.RequestType, "Expected request of type StorageFetchConsumers, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) request.Reply <- []string{"testgroup"} close(request.Reply) // Second request is a 404 request = <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchConsumers, request.RequestType, "Expected request of type StorageFetchConsumers, not %v", request.RequestType) assert.Equalf(t, "nocluster", request.Cluster, "Expected request Cluster to be nocluster, not %v", request.Cluster) close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/kafka/testcluster/consumer", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConsumerList err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, []string{"testgroup"}, resp.Consumers, "Expected Consumers list to contain just testgroup, not %v", resp.Consumers) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster/consumer", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_handleTopicDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchTopic, request.RequestType, "Expected request of type StorageFetchTopic, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "testtopic", request.Topic, "Expected request Topic to be testtopic, not %v", request.Topic) request.Reply <- []int64{345, 921} close(request.Reply) // Second request is a 404 request = <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchTopic, request.RequestType, "Expected request of type StorageFetchTopic, not %v", request.RequestType) assert.Equalf(t, "nocluster", request.Cluster, "Expected request Cluster to be nocluster, not %v", request.Cluster) assert.Equalf(t, "testtopic", request.Topic, "Expected request Topic to be testtopic, not %v", request.Topic) close(request.Reply) // Third request is a 404 request = <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchTopic, request.RequestType, "Expected request of type StorageFetchTopic, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "notopic", request.Topic, "Expected request Topic to be notopic, not %v", request.Topic) close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/kafka/testcluster/topic/testtopic", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseTopicDetail err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Equalf(t, []int64{345, 921}, resp.Offsets, "Expected Offsets list to contain [345, 921], not %v", resp.Offsets) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster/topic/testtopic", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) // Call a third time for a 404 req, err = http.NewRequest("GET", "/v3/kafka/testcluster/topic/notopic", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_handleConsumerDetail(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchConsumer, request.RequestType, "Expected request of type StorageFetchConsumer, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) response := make(protocol.ConsumerTopics) response["testtopic"] = make([]*protocol.ConsumerPartition, 1) response["testtopic"][0] = &protocol.ConsumerPartition{ Offsets: make([]*protocol.ConsumerOffset, 1), Owner: "somehost", CurrentLag: 2345, } response["testtopic"][0].Offsets[0] = &protocol.ConsumerOffset{ Offset: 9837458, Timestamp: 12837487, Lag: 2355, } request.Reply <- response close(request.Reply) // Second request is a 404 request = <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchConsumer, request.RequestType, "Expected request of type StorageFetchConsumer, not %v", request.RequestType) assert.Equalf(t, "nocluster", request.Cluster, "Expected request Cluster to be nocluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) close(request.Reply) // Third request is a 404 request = <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchConsumer, request.RequestType, "Expected request of type StorageFetchConsumer, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "nogroup", request.Group, "Expected request Group to be nogroup, not %v", request.Group) close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/kafka/testcluster/consumer/testgroup", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseConsumerDetail err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.Lenf(t, resp.Topics, 1, "Expected response to contain exactly one topic, not %v", len(resp.Topics)) topic, ok := resp.Topics["testtopic"] assert.True(t, ok, "Expected topic name to be testtopic") assert.Lenf(t, topic, 1, "Expected topic to contain exactly one partition, not %v", len(topic)) assert.Equalf(t, "somehost", topic[0].Owner, "Expected partition Owner to be somehost, not %v", topic[0].Owner) assert.Equalf(t, uint64(2345), topic[0].CurrentLag, "Expected partition CurrentLag to be 2345, not %v", topic[0].CurrentLag) assert.Lenf(t, topic[0].Offsets, 1, "Expected partition to have exactly one offset, not %v", topic[0].Offsets) assert.Equalf(t, int64(9837458), topic[0].Offsets[0].Offset, "Expected Offset to be 9837458, not %v", topic[0].Offsets[0].Offset) assert.Equalf(t, int64(12837487), topic[0].Offsets[0].Timestamp, "Expected Timestamp to be 12837487, not %v", topic[0].Offsets[0].Timestamp) assert.Equalf(t, uint64(2355), topic[0].Offsets[0].Lag, "Expected Lag to be 2355, not %v", topic[0].Offsets[0].Lag) // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster/consumer/testgroup", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) // Call a third time for a 404 req, err = http.NewRequest("GET", "/v3/kafka/testcluster/consumer/nogroup", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } // Custom response types for consumer status, as the status field will be a string type ResponsePartition struct { Topic string `json:"topic"` Partition int32 `json:"partition"` Status string `json:"status"` Start *protocol.ConsumerOffset `json:"start"` End *protocol.ConsumerOffset `json:"end"` CurrentLag int64 `json:"current_lag"` Complete float32 `json:"complete"` } type ResponseStatus struct { Cluster string `json:"cluster"` Group string `json:"group"` Status string `json:"status"` Complete float32 `json:"complete"` Partitions []*ResponsePartition `json:"partitions"` TotalPartitions int `json:"partition_count"` Maxlag *ResponsePartition `json:"maxlag"` TotalLag uint64 `json:"totallag"` } type ResponseType struct { Error bool `json:"error"` Message string `json:"message"` Status ResponseStatus `json:"status"` Request httpResponseRequestInfo `json:"request"` } func TestHttpServer_handleConsumerStatus(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected evaluator request go func() { request := <-coordinator.App.EvaluatorChannel assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) assert.False(t, request.ShowAll, "Expected request ShowAll to be False") response := &protocol.ConsumerGroupStatus{ Cluster: request.Cluster, Group: request.Group, Status: protocol.StatusOK, Complete: 1.0, Partitions: make([]*protocol.PartitionStatus, 0), TotalPartitions: 2134, Maxlag: &protocol.PartitionStatus{ Topic: "testtopic", Partition: 0, Status: protocol.StatusOK, Start: &protocol.ConsumerOffset{ Offset: 9836458, Timestamp: 12836487, Lag: 3254, }, End: &protocol.ConsumerOffset{ Offset: 9837458, Timestamp: 12837487, Lag: 2355, }, }, TotalLag: 2345, } request.Reply <- response close(request.Reply) // Second request is a 404 request = <-coordinator.App.EvaluatorChannel assert.Equalf(t, "nocluster", request.Cluster, "Expected request Cluster to be nocluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) assert.False(t, request.ShowAll, "Expected request ShowAll to be False") request.Reply <- &protocol.ConsumerGroupStatus{ Cluster: request.Cluster, Group: request.Group, Status: protocol.StatusNotFound, Complete: 1.0, Partitions: make([]*protocol.PartitionStatus, 0), Maxlag: nil, TotalLag: 0, } close(request.Reply) // Third request is a 404 request = <-coordinator.App.EvaluatorChannel assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "nogroup", request.Group, "Expected request Group to be nogroup, not %v", request.Group) assert.False(t, request.ShowAll, "Expected request ShowAll to be False") request.Reply <- &protocol.ConsumerGroupStatus{ Cluster: request.Cluster, Group: request.Group, Status: protocol.StatusNotFound, Complete: 1.0, Partitions: make([]*protocol.PartitionStatus, 0), Maxlag: nil, TotalLag: 0, } close(request.Reply) // Now we switch to expecting /lag requests request = <-coordinator.App.EvaluatorChannel assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) assert.True(t, request.ShowAll, "Expected request ShowAll to be True") response.Partitions = make([]*protocol.PartitionStatus, 1) response.Partitions[0] = response.Maxlag request.Reply <- response close(request.Reply) // Fifth request is a 404 request = <-coordinator.App.EvaluatorChannel assert.Equalf(t, "nocluster", request.Cluster, "Expected request Cluster to be nocluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) assert.True(t, request.ShowAll, "Expected request ShowAll to be True") request.Reply <- &protocol.ConsumerGroupStatus{ Cluster: request.Cluster, Group: request.Group, Status: protocol.StatusNotFound, Complete: 1.0, Partitions: make([]*protocol.PartitionStatus, 0), Maxlag: nil, TotalLag: 0, } close(request.Reply) // Sixth request is a 404 request = <-coordinator.App.EvaluatorChannel assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "nogroup", request.Group, "Expected request Group to be nogroup, not %v", request.Group) assert.True(t, request.ShowAll, "Expected request ShowAll to be True") request.Reply <- &protocol.ConsumerGroupStatus{ Cluster: request.Cluster, Group: request.Group, Status: protocol.StatusNotFound, Complete: 1.0, Partitions: make([]*protocol.PartitionStatus, 0), Maxlag: nil, TotalLag: 0, } close(request.Reply) }() // Set up a request req, err := http.NewRequest("GET", "/v3/kafka/testcluster/consumer/testgroup/status", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder := json.NewDecoder(rr.Body) var resp ResponseType err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.NotNil(t, resp.Status, "Expected Status to not be nil") assert.Equalf(t, float32(1.0), resp.Status.Complete, "Expected Complete to be 1.0, not %v", resp.Status.Complete) assert.Lenf(t, resp.Status.Partitions, 0, "Expected Status to contain exactly zero partitions, not %v", len(resp.Status.Partitions)) assert.Equalf(t, 2134, resp.Status.TotalPartitions, "Expected TotalPartitions to be 2134, not %v", resp.Status.TotalPartitions) assert.NotNil(t, resp.Status.Maxlag, "Expected Maxlag to not be nil") // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster/consumer/testgroup/status", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) // Call a third time for a 404 req, err = http.NewRequest("GET", "/v3/kafka/testcluster/consumer/nogroup/status", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) // Call for complete status (/lag endpoint) req, err = http.NewRequest("GET", "/v3/kafka/testcluster/consumer/testgroup/lag", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Parse response body decoder = json.NewDecoder(rr.Body) err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") assert.NotNil(t, resp.Status, "Expected Status to not be nil") assert.Equalf(t, float32(1.0), resp.Status.Complete, "Expected Complete to be 1.0, not %v", resp.Status.Complete) assert.Lenf(t, resp.Status.Partitions, 1, "Expected Status to contain exactly one partition, not %v", len(resp.Status.Partitions)) assert.Equalf(t, 2134, resp.Status.TotalPartitions, "Expected TotalPartitions to be 2134, not %v", resp.Status.TotalPartitions) assert.NotNil(t, resp.Status.Maxlag, "Expected Maxlag to not be nil") // Call again for a 404 req, err = http.NewRequest("GET", "/v3/kafka/nocluster/consumer/testgroup/lag", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) // Call a sixth time for a 404 req, err = http.NewRequest("GET", "/v3/kafka/testcluster/consumer/nogroup/lag", nil) assert.NoError(t, err, "Expected request setup to return no error") rr = httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusNotFound, rr.Code, "Expected response code to be 404, not %v", rr.Code) } func TestHttpServer_handleConsumerDelete(t *testing.T) { coordinator := fixtureConfiguredCoordinator() // Respond to the expected storage request go func() { request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageSetDeleteGroup, request.RequestType, "Expected request of type StorageFetchConsumer, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request Cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request Group to be testgroup, not %v", request.Group) // No response expected }() // Set up a request req, err := http.NewRequest("DELETE", "/v3/kafka/testcluster/consumer/testgroup", nil) assert.NoError(t, err, "Expected request setup to return no error") // Call the handler via httprouter rr := httptest.NewRecorder() coordinator.router.ServeHTTP(rr, req) assert.Equalf(t, http.StatusOK, rr.Code, "Expected response code to be 200, not %v", rr.Code) // Sleep briefly just to catch the goroutine above throwing a failure time.Sleep(100 * time.Millisecond) // Parse response body decoder := json.NewDecoder(rr.Body) var resp httpResponseError err = decoder.Decode(&resp) assert.NoError(t, err, "Expected body decode to return no error") assert.False(t, resp.Error, "Expected response Error to be false") } burrow-1.2.1/core/internal/httpserver/structs.go000066400000000000000000000230771343357346000220640ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package httpserver import "github.com/linkedin/Burrow/core/protocol" type logLevelRequest struct { Level string `json:"level"` } type httpResponseLogLevel struct { Error bool `json:"error"` Message string `json:"message"` Level string `json:"level"` Request httpResponseRequestInfo `json:"request"` } type httpResponseRequestInfo struct { URI string `json:"url"` Host string `json:"host"` } type httpResponseError struct { Error bool `json:"error"` Message string `json:"message"` Request httpResponseRequestInfo `json:"request"` } type httpResponseTLSProfile struct { Name string `json:"name"` NoVerify bool `json:"noverify"` CertFile string `json:"certfile"` KeyFile string `json:"keyfile"` CAFile string `json:"cafile"` } type httpResponseSASLProfile struct { Name string `json:"name"` HandshakeFirst bool `json:"handshake-first"` Username string `json:"username"` } type httpResponseClientProfile struct { Name string `json:"name"` ClientID string `json:"client-id"` KafkaVersion string `json:"kafka-version"` TLS *httpResponseTLSProfile `json:"tls"` SASL *httpResponseSASLProfile `json:"sasl"` } type httpResponseClusterList struct { Error bool `json:"error"` Message string `json:"message"` Clusters []string `json:"clusters"` Request httpResponseRequestInfo `json:"request"` } type httpResponseTopicList struct { Error bool `json:"error"` Message string `json:"message"` Topics []string `json:"topics"` Request httpResponseRequestInfo `json:"request"` } type httpResponseTopicDetail struct { Error bool `json:"error"` Message string `json:"message"` Offsets []int64 `json:"offsets"` Request httpResponseRequestInfo `json:"request"` } type httpResponseTopicConsumerDetail struct { Error bool `json:"error"` Message string `json:"message"` Consumers []string `json:"consumers"` Request httpResponseRequestInfo `json:"request"` } type httpResponseConsumerList struct { Error bool `json:"error"` Message string `json:"message"` Consumers []string `json:"consumers"` Request httpResponseRequestInfo `json:"request"` } type httpResponseConsumerDetail struct { Error bool `json:"error"` Message string `json:"message"` Topics protocol.ConsumerTopics `json:"topics"` Request httpResponseRequestInfo `json:"request"` } type httpResponseConsumerStatus struct { Error bool `json:"error"` Message string `json:"message"` Status protocol.ConsumerGroupStatus `json:"status"` Request httpResponseRequestInfo `json:"request"` } type httpResponseConfigGeneral struct { PIDFile string `json:"pidfile"` StdoutLogfile string `json:"stdout-logfile"` AccessControlAllowOrigin string `json:"access-control-allow-origin"` } type httpResponseConfigLogging struct { Filename string `json:"filename"` MaxSize int `json:"max-size"` MaxBackups int `json:"max-backups"` MaxAge int `json:"max-age"` UseLocalTime bool `json:"use-local-time"` UseCompression bool `json:"use-compression"` Level string `json:"level"` } type httpResponseConfigZookeeper struct { Servers []string `json:"servers"` Timeout int `json:"timeout"` RootPath string `json:"root-path"` } type httpResponseConfigHTTPServer struct { Address string `json:"address"` TLS string `json:"tls"` Timeout int `json:"timeout"` } type httpResponseConfigMain struct { Error bool `json:"error"` Message string `json:"message"` Request httpResponseRequestInfo `json:"request"` General httpResponseConfigGeneral `json:"general"` Logging httpResponseConfigLogging `json:"logging"` Zookeeper httpResponseConfigZookeeper `json:"zookeeper"` HTTPServer map[string]httpResponseConfigHTTPServer `json:"httpserver"` } type httpResponseConfigModuleList struct { Error bool `json:"error"` Message string `json:"message"` Request httpResponseRequestInfo `json:"request"` Coordinator string `json:"coordinator"` Modules []string `json:"modules"` } type httpResponseConfigModuleDetail struct { Error bool `json:"error"` Message string `json:"message"` Module interface{} `json:"module"` Request httpResponseRequestInfo `json:"request"` } type httpResponseConfigModuleStorage struct { ClassName string `json:"class-name"` Intervals int `json:"intervals"` MinDistance int64 `json:"min-distance"` GroupWhitelist string `json:"group-whitelist"` ExpireGroup int64 `json:"expire-group"` } type httpResponseConfigModuleCluster struct { ClassName string `json:"class-name"` Servers []string `json:"servers"` ClientProfile httpResponseClientProfile `json:"client-profile"` TopicRefresh int64 `json:"topic-refresh"` OffsetRefresh int64 `json:"offset-refresh"` } type httpResponseConfigModuleConsumer struct { ClassName string `json:"class-name"` Cluster string `json:"cluster"` Servers []string `json:"servers"` GroupWhitelist string `json:"group-whitelist"` ZookeeperPath string `json:"zookeeper-path"` ZookeeperTimeout int32 `json:"zookeeper-timeout"` ClientProfile httpResponseClientProfile `json:"client-profile"` OffsetsTopic string `json:"offsets-topic"` StartLatest bool `json:"start-latest"` } type httpResponseConfigModuleEvaluator struct { ClassName string `json:"class-name"` ExpireCache int64 `json:"expire-cache"` } type httpResponseConfigModuleNotifierHTTP struct { ClassName string `json:"class-name"` GroupWhitelist string `json:"group-whitelist"` Interval int64 `json:"interval"` Threshold int `json:"threshold"` Timeout int `json:"timeout"` Keepalive int `json:"keepalive"` URLOpen string `json:"url-open"` URLClose string `json:"url-close"` MethodOpen string `json:"method-open"` MethodClose string `json:"method-close"` TemplateOpen string `json:"template-open"` TemplateClose string `json:"template-close"` Extras map[string]string `json:"extra"` SendClose bool `json:"send-close"` } type httpResponseConfigModuleNotifierSlack struct { ClassName string `json:"class-name"` GroupWhitelist string `json:"group-whitelist"` Interval int64 `json:"interval"` Threshold int `json:"threshold"` Timeout int `json:"timeout"` Keepalive int `json:"keepalive"` TemplateOpen string `json:"template-open"` TemplateClose string `json:"template-close"` Extras map[string]string `json:"extra"` SendClose bool `json:"send-close"` Channel string `json:"channel"` Username string `json:"username"` IconURL string `json:"icon-url"` IconEmoji string `json:"icon-emoji"` } type httpResponseConfigModuleNotifierEmail struct { ClassName string `json:"class-name"` GroupWhitelist string `json:"group-whitelist"` Interval int64 `json:"interval"` Threshold int `json:"threshold"` TemplateOpen string `json:"template-open"` TemplateClose string `json:"template-close"` Extras map[string]string `json:"extra"` SendClose bool `json:"send-close"` Server string `json:"server"` Port int `json:"port"` AuthType string `json:"auth-type"` Username string `json:"username"` From string `json:"from"` To string `json:"to"` } type httpResponseConfigModuleNotifierNull struct { ClassName string `json:"class-name"` GroupWhitelist string `json:"group-whitelist"` Interval int64 `json:"interval"` Threshold int `json:"threshold"` TemplateOpen string `json:"template-open"` TemplateClose string `json:"template-close"` Extras map[string]string `json:"extra"` SendClose bool `json:"send-close"` } burrow-1.2.1/core/internal/notifier/000077500000000000000000000000001343357346000174265ustar00rootroot00000000000000burrow-1.2.1/core/internal/notifier/coordinator.go000066400000000000000000000441341343357346000223060ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package notifier - Status notification subsystem. // The notifier subsystem watches the status for all consumer groups and uses the configured modules to send // information about the status of those groups to outside systems, such as via email or calls to HTTP endpoints. The // message bodies are built using templates, and notifications can be sent for both active problems as well as when // those problems close. // // Modules // // Currently, the following modules are provided: // // * email - Send an email // // * http - Call a remote HTTP endpoint // // * null - This is a no-op notifier that is used for testing only package notifier import ( "errors" "math" "math/rand" "regexp" "sync" "text/template" "time" "github.com/pborman/uuid" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // Module defines a means of sending out notifications of consumer group status (such as email), as well as regular // expressions describing what groups to notify for. The module itself only provides the logic for how to send a // notification in the Notify func - timing loops, and handling requests for group evaluation, are handled in the // coordinator centrally. type Module interface { protocol.Module GetName() string GetGroupWhitelist() *regexp.Regexp GetGroupBlacklist() *regexp.Regexp GetLogger() *zap.Logger AcceptConsumerGroup(*protocol.ConsumerGroupStatus) bool Notify(*protocol.ConsumerGroupStatus, string, time.Time, bool) } type consumerGroup struct { ID string Start time.Time LastNotify map[string]time.Time LastEval time.Time } type clusterGroups struct { Groups map[string]*consumerGroup Lock *sync.RWMutex } // Coordinator manages all notifier modules, making sure they are configured, started, and stopped at the // appropriate time. Unlike other coordinators, it also performs a significant amount of ongoing work. type Coordinator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger modules map[string]protocol.Module minInterval int64 groupRefresh helpers.Ticker doEvaluations bool evaluatorResponse chan *protocol.ConsumerGroupStatus running sync.WaitGroup quitChannel chan struct{} templateParseFunc func(...string) (*template.Template, error) notifyModuleFunc func(Module, *protocol.ConsumerGroupStatus, time.Time, string) clusters map[string]*clusterGroups clusterLock *sync.RWMutex } // getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there // is any error, it will panic with an appropriate message describing the problem. func getModuleForClass(app *protocol.ApplicationContext, moduleName string, className string, groupWhitelist *regexp.Regexp, groupBlacklist *regexp.Regexp, extras map[string]string, templateOpen *template.Template, templateClose *template.Template) protocol.Module { logger := app.Logger.With( zap.String("type", "module"), zap.String("coordinator", "notifier"), zap.String("class", className), zap.String("name", moduleName), ) switch className { case "http": return &HTTPNotifier{ App: app, Log: logger, groupWhitelist: groupWhitelist, groupBlacklist: groupBlacklist, extras: extras, templateOpen: templateOpen, templateClose: templateClose, } case "email": return &EmailNotifier{ App: app, Log: logger, groupWhitelist: groupWhitelist, groupBlacklist: groupBlacklist, extras: extras, templateOpen: templateOpen, templateClose: templateClose, } case "null": return &NullNotifier{ App: app, Log: logger, groupWhitelist: groupWhitelist, groupBlacklist: groupBlacklist, extras: extras, templateOpen: templateOpen, templateClose: templateClose, } default: panic("Unknown notifier className provided: " + className) } } // Configure is called to create each of the configured notifier modules and call their Configure funcs to validate // their individual configurations and set them up. If there are any problems, it is expected that these funcs will // panic with a descriptive error message, as configuration failures are not recoverable errors. func (nc *Coordinator) Configure() { nc.Log.Info("configuring") nc.modules = make(map[string]protocol.Module) nc.clusters = make(map[string]*clusterGroups) nc.clusterLock = &sync.RWMutex{} nc.minInterval = math.MaxInt64 nc.quitChannel = make(chan struct{}) nc.running = sync.WaitGroup{} nc.evaluatorResponse = make(chan *protocol.ConsumerGroupStatus) // Set the function for parsing templates and calling module Notify (configurable to enable testing) if nc.templateParseFunc == nil { nc.templateParseFunc = func(filenames ...string) (*template.Template, error) { return template.New("notifier").Funcs(helperFunctionMap).ParseFiles(filenames...) } } if nc.notifyModuleFunc == nil { nc.notifyModuleFunc = nc.notifyModule } // Create all configured notifier modules, add to list of notifier // Note - we do a lot more work here than for other coordinators. This is because the notifier modules really just // contain the logic to send the notification. Many of the parts, such as the whitelist and templates, are // common to all notifier modules for name := range viper.GetStringMap("notifier") { configRoot := "notifier." + name // Set some defaults for common module fields viper.SetDefault(configRoot+".interval", 60) viper.SetDefault(configRoot+".send-interval", viper.GetInt64(configRoot+".interval")) viper.SetDefault(configRoot+".threshold", 2) // Compile the whitelist for the consumer groups to notify for var groupWhitelist *regexp.Regexp whitelist := viper.GetString(configRoot + ".group-whitelist") if whitelist != "" { re, err := regexp.Compile(whitelist) if err != nil { nc.Log.Panic("Failed to compile group whitelist", zap.String("module", name)) panic(err) } groupWhitelist = re } // Compile the blacklist for the consumer groups to not notify for var groupBlacklist *regexp.Regexp blacklist := viper.GetString(configRoot + ".group-blacklist") if blacklist != "" { re, err := regexp.Compile(blacklist) if err != nil { nc.Log.Panic("Failed to compile group blacklist", zap.String("module", name)) panic(err) } groupBlacklist = re } // Set up extra fields for the templates extras := viper.GetStringMapString(configRoot + ".extras") // Compile the templates var templateOpen, templateClose *template.Template tmpl, err := nc.templateParseFunc(viper.GetString(configRoot + ".template-open")) if err != nil { nc.Log.Panic("Failed to compile TemplateOpen", zap.Error(err), zap.String("module", name)) panic(err) } templateOpen = tmpl.Templates()[0] if viper.GetBool(configRoot + ".send-close") { tmpl, err = nc.templateParseFunc(viper.GetString(configRoot + ".template-close")) if err != nil { nc.Log.Panic("Failed to compile TemplateClose", zap.Error(err), zap.String("module", name)) panic(err) } templateClose = tmpl.Templates()[0] } module := getModuleForClass(nc.App, name, viper.GetString(configRoot+".class-name"), groupWhitelist, groupBlacklist, extras, templateOpen, templateClose) module.Configure(name, configRoot) nc.modules[name] = module interval := viper.GetInt64(configRoot + ".interval") if interval < nc.minInterval { nc.minInterval = interval } } // If there are no modules specified, the minInterval will still be MaxInt64. Set it to a large number of seconds if nc.minInterval == math.MaxInt64 { nc.minInterval = 310536000 } // Set up the tickers but do not start them // TODO - should probably be configurable nc.groupRefresh = helpers.NewPausableTicker(60 * time.Second) } // Start calls each of the configured notifier modules' underlying Start funcs. If any module Start returns an error, // this func stops immediately and returns that error to the caller. No further modules will be loaded after that. // // We also start a timer to periodically update the list of known clusters and consumer groups, as well as the // goroutine which manages whether or not we are performing group evaluation requests. We also start the responseLoop // func that handles all evaluation replies and calls the module Notify methods as appropriate. func (nc *Coordinator) Start() error { nc.Log.Info("starting") // The notifier coordinator is responsible for fetching group evaluations and handing them off to the individual // notifier modules. nc.running.Add(1) go nc.responseLoop() err := helpers.StartCoordinatorModules(nc.modules) if err != nil { return errors.New("Error starting notifier module: " + err.Error()) } // Run the group refresh regardless of whether or not we're sending notifications nc.groupRefresh.Start() // Run a goroutine to manage whether or not we're performing evaluations go nc.manageEvalLoop() // Run our main loop to watch tickers and take actions nc.running.Add(1) go nc.tickerLoop() return nil } // Stop stops the group refresh ticker, and causes the evaluation response handler and the evaluation request manager to // both stop. It then calls each of the configured notifier modules' underlying Stop funcs. It is expected that the // module Stop will not return until the module has been completely stopped. While an error can be returned, this func // always returns no error, as a failure during stopping is not a critical failure func (nc *Coordinator) Stop() error { nc.Log.Info("stopping") nc.groupRefresh.Stop() nc.doEvaluations = false close(nc.quitChannel) // The individual notifier modules can choose whether or not to implement a wait in the Stop routine helpers.StopCoordinatorModules(nc.modules) return nil } func (nc *Coordinator) manageEvalLoop() { lock := nc.App.Zookeeper.NewLock(nc.App.ZookeeperRoot + "/notifier") for { time.Sleep(100 * time.Millisecond) err := lock.Lock() if err != nil { nc.Log.Warn("failed to get zk lock", zap.Error(err)) continue } // We've got the lock, start the evaluation loop nc.doEvaluations = true nc.running.Add(1) go nc.sendEvaluatorRequests() nc.Log.Info("starting evaluations", zap.Error(err)) // Wait for ZK session expiration, and stop doing evaluations if it happens nc.App.ZookeeperExpired.L.Lock() nc.App.ZookeeperExpired.Wait() nc.App.ZookeeperExpired.L.Unlock() nc.doEvaluations = false nc.Log.Info("stopping evaluations", zap.Error(err)) // Wait for the ZK connection to come back before trying again for !nc.App.ZookeeperConnected { time.Sleep(100 * time.Millisecond) } } } // We keep this function trivial because tickers are not easy to mock/test in golang func (nc *Coordinator) tickerLoop() { defer nc.running.Done() for { select { case <-nc.groupRefresh.GetChannel(): nc.sendClusterRequest() case <-nc.quitChannel: return } } } func (nc *Coordinator) sendClusterRequest() { // Send a request to the storage module for a list of clusters, and spawn a goroutine to process it request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchClusters, Reply: make(chan interface{}), } nc.running.Add(1) go nc.processClusterList(request.Reply) helpers.TimeoutSendStorageRequest(nc.App.StorageChannel, request, 1) } func (nc *Coordinator) sendEvaluatorRequests() { defer nc.running.Done() for nc.doEvaluations { // Loop through all clusters and groups and send any evaluation requests that are due timeNow := time.Now() sendBefore := timeNow.Add(-time.Duration(nc.minInterval) * time.Second) // Fire off evaluation requests for every group we know about nc.clusterLock.RLock() for cluster, consumerGroup := range nc.clusters { consumerGroup.Lock.RLock() for consumer, groupInfo := range consumerGroup.Groups { if groupInfo.LastEval.Before(sendBefore) { nc.Log.Debug("Evaluating group", zap.String("group", consumer)) go func(sendCluster string, sendConsumer string) { nc.App.EvaluatorChannel <- &protocol.EvaluatorRequest{ Reply: nc.evaluatorResponse, Cluster: sendCluster, Group: sendConsumer, } }(cluster, consumer) groupInfo.LastEval = timeNow } } consumerGroup.Lock.RUnlock() } nc.clusterLock.RUnlock() // Sleep briefly to prevent a tight loop time.Sleep(time.Millisecond) } } func (nc *Coordinator) responseLoop() { defer nc.running.Done() for { select { case response := <-nc.evaluatorResponse: // If response is nil, the group no longer exists if response == nil { continue } // As long as the response is not NotFound, send it to the modules if response.Status != protocol.StatusNotFound { nc.running.Add(1) go nc.checkAndSendResponseToModules(response) } case <-nc.quitChannel: return } } } func (nc *Coordinator) checkAndSendResponseToModules(response *protocol.ConsumerGroupStatus) { defer nc.running.Done() nc.clusterLock.RLock() defer nc.clusterLock.RUnlock() cluster := nc.clusters[response.Cluster] cluster.Lock.RLock() defer cluster.Lock.RUnlock() cgroup, ok := cluster.Groups[response.Group] if !ok { // The group must have just been deleted return } if cgroup.Start.IsZero() && (response.Status > protocol.StatusOK) { // New incident - assign an ID and start time cgroup.ID = uuid.NewRandom().String() cgroup.Start = time.Now() } for _, genericModule := range nc.modules { module := genericModule.(Module) // No whitelist means everything passes groupWhitelist := module.GetGroupWhitelist() groupBlacklist := module.GetGroupBlacklist() if (groupWhitelist != nil) && (!groupWhitelist.MatchString(response.Group)) { continue } if (groupBlacklist != nil) && groupBlacklist.MatchString(response.Group) { continue } if module.AcceptConsumerGroup(response) { nc.running.Add(1) nc.notifyModuleFunc(module, response, cgroup.Start, cgroup.ID) } } if response.Status == protocol.StatusOK { // Incident closed - clear the start time and event ID cgroup.ID = "" cgroup.Start = time.Time{} } } func (nc *Coordinator) processClusterList(replyChan chan interface{}) { defer nc.running.Done() response := <-replyChan clusterList, _ := response.([]string) requestMap := make(map[string]*protocol.StorageRequest) nc.clusterLock.Lock() for _, cluster := range clusterList { // Make sure we have a map entry for the cluster if _, ok := nc.clusters[cluster]; !ok { nc.clusters[cluster] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } } // Create a request for the group list for this cluster requestMap[cluster] = &protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumers, Reply: make(chan interface{}), Cluster: cluster, } } // Delete clusters that no longer exist for cluster := range nc.clusters { if _, ok := requestMap[cluster]; !ok { delete(nc.clusters, cluster) } } nc.clusterLock.Unlock() // Fire off requests for group lists to the storage module, with goroutines to process the responses for cluster, request := range requestMap { nc.running.Add(1) go nc.processConsumerList(cluster, request.Reply) helpers.TimeoutSendStorageRequest(nc.App.StorageChannel, request, 1) } } func (nc *Coordinator) processConsumerList(cluster string, replyChan chan interface{}) { defer nc.running.Done() response := <-replyChan consumerList, _ := response.([]string) nc.clusterLock.RLock() defer nc.clusterLock.RUnlock() if _, ok := nc.clusters[cluster]; ok { nc.clusters[cluster].Lock.Lock() consumerMap := make(map[string]struct{}) for _, group := range consumerList { consumerMap[group] = struct{}{} if _, ok := nc.clusters[cluster].Groups[group]; !ok { nc.clusters[cluster].Groups[group] = &consumerGroup{ LastNotify: make(map[string]time.Time), LastEval: time.Now().Add(-time.Duration(rand.Int63n(nc.minInterval*1000)) * time.Millisecond), } } } // Use the map we just made to delete consumers that no longer exist for group := range nc.clusters[cluster].Groups { if _, ok := consumerMap[group]; !ok { delete(nc.clusters[cluster].Groups, group) } } nc.clusters[cluster].Lock.Unlock() } } func (nc *Coordinator) notifyModule(module Module, status *protocol.ConsumerGroupStatus, startTime time.Time, eventID string) { defer nc.running.Done() // Note - it is assumed that a read lock is already held when calling notifyModule cgroup, ok := nc.clusters[status.Cluster].Groups[status.Group] if !ok { // The group must have just been deleted return } // Closed incidents get sent regardless of the threshold for the module moduleName := module.GetName() if (!startTime.IsZero()) && (status.Status == protocol.StatusOK) && viper.GetBool("notifier."+moduleName+".send-close") { module.Notify(status, eventID, startTime, true) cgroup.LastNotify[module.GetName()] = time.Time{} return } // Only send a notification if the current status is above the module's threshold if int(status.Status) < viper.GetInt("notifier."+module.GetName()+".threshold") { return } // Only send the notification if it's been at least our Interval since the last one for this group currentTime := time.Now() if currentTime.Sub(cgroup.LastNotify[module.GetName()]) > (time.Duration(viper.GetInt("notifier."+moduleName+".send-interval")) * time.Second) { module.Notify(status, eventID, startTime, false) cgroup.LastNotify[module.GetName()] = currentTime } } burrow-1.2.1/core/internal/notifier/coordinator_race_test.go000066400000000000000000000125601343357346000243350ustar00rootroot00000000000000// +build !race /* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "time" "github.com/stretchr/testify/assert" "testing" "github.com/linkedin/Burrow/core/internal/helpers" "sync" ) // This tests the full set of calls to send evaluator requests. It triggers the race detector because of setting // doEvaluations to false to end the loop. func TestCoordinator_sendEvaluatorRequests(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // A test cluster and group to send requests for coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ LastNotify: make(map[string]time.Time), LastEval: time.Now().Add(-time.Duration(coordinator.minInterval) * time.Second), } coordinator.clusters["testcluster2"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster2"].Groups["testgroup2"] = &consumerGroup{ LastNotify: make(map[string]time.Time), LastEval: time.Now().Add(-time.Duration(coordinator.minInterval) * time.Second), } coordinator.doEvaluations = true coordinator.running.Add(1) go coordinator.sendEvaluatorRequests() // We expect to get 2 requests for i := 0; i < 2; i++ { request := <-coordinator.App.EvaluatorChannel switch request.Cluster { case "testcluster": assert.Equalf(t, "testcluster", request.Cluster, "Expected request cluster to be testcluster, not %v", request.Cluster) assert.Equalf(t, "testgroup", request.Group, "Expected request group to be testgroup, not %v", request.Group) assert.False(t, request.ShowAll, "Expected ShowAll to be false") case "testcluster2": assert.Equalf(t, "testcluster2", request.Cluster, "Expected request cluster to be testcluster2, not %v", request.Cluster) assert.Equalf(t, "testgroup2", request.Group, "Expected request group to be testgroup2, not %v", request.Group) assert.False(t, request.ShowAll, "Expected ShowAll to be false") default: assert.Failf(t, "Received unexpected request for cluster %v, group %v", request.Cluster, request.Group) } } select { case <-coordinator.App.EvaluatorChannel: assert.Fail(t, "Received extra request on the evaluator channel") default: // All is good - we didn't expect to find another request } coordinator.doEvaluations = false } // We know this will trigger the race detector, because of the way we manipulate the ZK state func TestCoordinator_manageEvalLoop_Start(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // Add mock calls for the Zookeeper client - Lock immediately returns with no error mockLock := &helpers.MockZookeeperLock{} mockLock.On("Lock").Return(nil) mockZk := coordinator.App.Zookeeper.(*helpers.MockZookeeperClient) mockZk.On("NewLock", "/burrow/notifier").Return(mockLock) go coordinator.manageEvalLoop() time.Sleep(200 * time.Millisecond) mockLock.AssertExpectations(t) mockZk.AssertExpectations(t) assert.True(t, coordinator.doEvaluations, "Expected doEvaluations to be true") } // We know this will trigger the race detector, because of the way we manipulate the ZK state func TestCoordinator_manageEvalLoop_Expiration(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // Add mock calls for the Zookeeper client - Lock immediately returns with no error mockLock := &helpers.MockZookeeperLock{} mockLock.On("Lock").Return(nil) mockZk := coordinator.App.Zookeeper.(*helpers.MockZookeeperClient) mockZk.On("NewLock", "/burrow/notifier").Return(mockLock) go coordinator.manageEvalLoop() time.Sleep(200 * time.Millisecond) // ZK gets disconnected and expired coordinator.App.ZookeeperConnected = false coordinator.App.ZookeeperExpired.Broadcast() time.Sleep(300 * time.Millisecond) mockLock.AssertExpectations(t) mockZk.AssertExpectations(t) assert.False(t, coordinator.doEvaluations, "Expected doEvaluations to be false") } // We know this will trigger the race detector, because of the way we manipulate the ZK state func TestCoordinator_manageEvalLoop_Reconnect(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // Add mock calls for the Zookeeper client - Lock immediately returns with no error mockLock := &helpers.MockZookeeperLock{} mockLock.On("Lock").Return(nil) mockZk := coordinator.App.Zookeeper.(*helpers.MockZookeeperClient) mockZk.On("NewLock", "/burrow/notifier").Return(mockLock) go coordinator.manageEvalLoop() time.Sleep(200 * time.Millisecond) // ZK gets disconnected and expired coordinator.App.ZookeeperConnected = false coordinator.App.ZookeeperExpired.Broadcast() time.Sleep(200 * time.Millisecond) // ZK gets reconnected coordinator.App.ZookeeperConnected = true time.Sleep(300 * time.Millisecond) mockLock.AssertExpectations(t) mockZk.AssertExpectations(t) assert.True(t, coordinator.doEvaluations, "Expected doEvaluations to be true") } burrow-1.2.1/core/internal/notifier/coordinator_test.go000066400000000000000000000563021343357346000233450ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "errors" "fmt" "regexp" "sync" "text/template" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) func fixtureCoordinator() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), StorageChannel: make(chan *protocol.StorageRequest), EvaluatorChannel: make(chan *protocol.EvaluatorRequest), Zookeeper: &helpers.MockZookeeperClient{}, ZookeeperRoot: "/burrow", ZookeeperConnected: true, ZookeeperExpired: &sync.Cond{L: &sync.Mutex{}}, } // Simple parser replacement that returns a blank template coordinator.templateParseFunc = func(filenames ...string) (*template.Template, error) { return template.New("test").Parse("") } viper.Reset() viper.Set("notifier.test.class-name", "null") viper.Set("notifier.test.group-whitelist", ".*") viper.Set("notifier.test.threshold", 1) viper.Set("notifier.test.interval", 5) viper.Set("notifier.test.timeout", 2) viper.Set("notifier.test.keepalive", 10) viper.Set("notifier.test.template-open", "template_open") viper.Set("notifier.test.template-close", "template_close") viper.Set("notifier.test.extras.foo", "bar") viper.Set("notifier.test.send-close", false) return &coordinator } func TestCoordinator_ImplementsCoordinator(t *testing.T) { assert.Implements(t, (*protocol.Coordinator)(nil), new(Coordinator)) } func TestCoordinator_Configure(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) module := coordinator.modules["test"].(*NullNotifier) assert.True(t, module.CalledConfigure, "Expected module Configure to be called") assert.Lenf(t, module.extras, 1, "Expected exactly one extra entry to be set, not %v", len(module.extras)) val, ok := module.extras["foo"] assert.True(t, ok, "Expected extras key to be 'foo'") assert.Equalf(t, "bar", val, "Expected value of extras 'foo' to be 'bar', not %v", val) } func TestCoordinator_Configure_NoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Reset() coordinator.Configure() assert.Equalf(t, int64(310536000), coordinator.minInterval, "Expected coordinator minInterval to be 310536000, not %v", coordinator.minInterval) } func TestCoordinator_Configure_TwoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("notifier.anothertest.class-name", "null") coordinator.Configure() assert.Lenf(t, coordinator.modules, 2, "Expected 2 modules configured, not %v", len(coordinator.modules)) module := coordinator.modules["test"].(*NullNotifier) assert.True(t, module.CalledConfigure, "Expected module 'test' Configure to be called") module = coordinator.modules["anothertest"].(*NullNotifier) assert.True(t, module.CalledConfigure, "Expected module 'anothertest' Configure to be called") } func TestCoordinator_Configure_BadRegexp(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("notifier.test.group-whitelist", "[") assert.Panics(t, func() { coordinator.Configure() }, "The code did not panic") } func TestCoordinator_Configure_BadTemplate(t *testing.T) { coordinator := fixtureCoordinator() // Simple parser replacement that returns an error coordinator.templateParseFunc = func(filenames ...string) (*template.Template, error) { return nil, errors.New("bad template") } assert.Panics(t, func() { coordinator.Configure() }, "The code did not panic") } func TestCoordinator_Configure_SendClose(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("notifier.test.send-close", true) // Simple parser replacement that returns an empty template using a string, or throws an error on unexpected use for this test coordinator.templateParseFunc = func(filenames ...string) (*template.Template, error) { if len(filenames) != 1 { return nil, errors.New("expected exactly 1 filename") } return template.New("test").Parse("") } coordinator.Configure() module := coordinator.modules["test"].(*NullNotifier) assert.NotNil(t, module.templateOpen, "Expected templateOpen to be set with a template") assert.NotNil(t, module.templateClose, "Expected templateClose to be set with a template") } func TestCoordinator_StartStop(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // Swap out the coordinator modules with a mock for testing mockModule := &helpers.MockModule{} mockModule.On("Start").Return(nil) mockModule.On("Stop").Return(nil) coordinator.modules["test"] = mockModule // Add mock calls for the Zookeeper client - We're not testing this part here, so just hang forever mockLock := &helpers.MockZookeeperLock{} mockLock.On("Lock").After(1 * time.Minute).Return(errors.New("neverworks")) mockZk := coordinator.App.Zookeeper.(*helpers.MockZookeeperClient) mockZk.On("NewLock", "/burrow/notifier").Return(mockLock) coordinator.Start() mockModule.AssertCalled(t, "Start") coordinator.Stop() mockModule.AssertCalled(t, "Stop") } // This tests the full set of calls to send and process storage requests func TestCoordinator_sendClusterRequest(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() // This cluster will get deleted coordinator.clusters["deleteme"] = &clusterGroups{} // This goroutine will receive the storage request for a cluster list, and respond with an appropriate list wg := &sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchClusters, request.RequestType, "Expected request type to be StorageFetchClusters, not %v", request.RequestType) // Send a response back with a cluster list, which will trigger another storage request request.Reply <- []string{"testcluster"} // This goroutine will receive the storage request for a consumer list, and respond with an appropriate list // We don't start it until after the first request is received, because otherwise we'll have a race condition wg.Add(1) go func() { defer wg.Done() request := <-coordinator.App.StorageChannel assert.Equalf(t, protocol.StorageFetchConsumers, request.RequestType, "Expected request type to be StorageFetchConsumers, not %v", request.RequestType) assert.Equalf(t, "testcluster", request.Cluster, "Expected request cluster to be testcluster, not %v", request.RequestType) // Send back the group list response, and sleep for a moment after that (otherwise we'll be racing the goroutine that updates the cluster groups request.Reply <- []string{"testgroup"} }() }() coordinator.sendClusterRequest() coordinator.running.Wait() wg.Wait() assert.Lenf(t, coordinator.clusters, 1, "Expected 1 entry in the clusters map, not %v", len(coordinator.clusters)) cluster, ok := coordinator.clusters["testcluster"] assert.True(t, ok, "Expected clusters map key to be testcluster") assert.Lenf(t, cluster.Groups, 1, "Expected 1 entry in the group list, not %v", len(cluster.Groups)) _, ok = cluster.Groups["testgroup"] assert.True(t, ok, "Expected group to be testgroup") } // Note, we do not check the calls to the module here, just that the response loop sets the event properly func TestCoordinator_responseLoop_NotFound(t *testing.T) { coordinator := fixtureCoordinator() // For NotFound, we expect the notifier will not be called at all coordinator.notifyModuleFunc = func(module Module, status *protocol.ConsumerGroupStatus, startTime time.Time, eventId string) { defer coordinator.running.Done() assert.Fail(t, "Expected notifyModule to not be called") } coordinator.Configure() // A test cluster and group to receive response for coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ LastNotify: make(map[string]time.Time), } // Test a NotFound response - we shouldn't do anything here responseNotFound := &protocol.ConsumerGroupStatus{ Cluster: "testcluster", Group: "testgroup", Status: protocol.StatusNotFound, } go func() { coordinator.evaluatorResponse <- responseNotFound // After a short wait, close the quit channel to release the responseLoop time.Sleep(100 * time.Millisecond) close(coordinator.quitChannel) }() coordinator.running.Add(1) coordinator.responseLoop() coordinator.running.Wait() close(coordinator.evaluatorResponse) time.Sleep(100 * time.Millisecond) group := coordinator.clusters["testcluster"].Groups["testgroup"] assert.Equalf(t, "", group.ID, "Expected group incident ID to be empty, not %v", group.ID) assert.True(t, group.Start.IsZero(), "Expected group incident start time to be unset") assert.True(t, group.LastNotify["test"].IsZero(), "Expected group last time to be unset") } func TestCoordinator_responseLoop_NoIncidentOK(t *testing.T) { coordinator := fixtureCoordinator() // We expect notifyModule to be called for the Null module in all cases (except NotFound) responseOK := &protocol.ConsumerGroupStatus{ Cluster: "testcluster", Group: "testgroup", Status: protocol.StatusOK, } coordinator.notifyModuleFunc = func(module Module, status *protocol.ConsumerGroupStatus, startTime time.Time, eventId string) { defer coordinator.running.Done() assert.Equal(t, "test", module.GetName(), "Expected to be called with the null notifier module") assert.Equal(t, responseOK, status, "Expected to be called with responseOK as the status") assert.True(t, startTime.IsZero(), "Expected to be called with zero value startTime") assert.Equal(t, "", eventId, "Expected to be called with empty eventId") } coordinator.Configure() // A test cluster and group to receive response for coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ LastNotify: make(map[string]time.Time), } // Test an OK response go func() { coordinator.evaluatorResponse <- responseOK // After a short wait, close the quit channel to release the responseLoop time.Sleep(100 * time.Millisecond) close(coordinator.quitChannel) }() coordinator.running.Add(1) coordinator.responseLoop() coordinator.running.Wait() close(coordinator.evaluatorResponse) time.Sleep(100 * time.Millisecond) group := coordinator.clusters["testcluster"].Groups["testgroup"] assert.Equalf(t, "", group.ID, "Expected group incident ID to be empty, not %v", group.ID) assert.True(t, group.Start.IsZero(), "Expected group incident start time to be unset") assert.True(t, group.LastNotify["test"].IsZero(), "Expected group last time to be unset") module := coordinator.modules["test"].(*NullNotifier) assert.True(t, module.CalledAcceptConsumerGroup, "Expected module 'test' AcceptConsumerGroup to be called") } func TestCoordinator_responseLoop_HaveIncidentOK(t *testing.T) { coordinator := fixtureCoordinator() // We expect notifyModule to be called for the Null module in all cases (except NotFound) mockStartTime, _ := time.Parse(time.RFC3339, "2012-11-01T22:08:41+00:00") responseOK := &protocol.ConsumerGroupStatus{ Cluster: "testcluster", Group: "testgroup", Status: protocol.StatusOK, } coordinator.notifyModuleFunc = func(module Module, status *protocol.ConsumerGroupStatus, startTime time.Time, eventId string) { defer coordinator.running.Done() assert.Equal(t, "test", module.GetName(), "Expected to be called with the null notifier module") assert.Equal(t, responseOK, status, "Expected to be called with responseOK as the status") assert.Equal(t, mockStartTime, startTime, "Expected to be called with mockStartTime as the startTime") assert.Equalf(t, "testidstring", eventId, "Expected to be called with eventId as 'testidstring', not %v", eventId) } coordinator.Configure() // A test cluster and group to receive response for, with the incident active coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ Start: mockStartTime, ID: "testidstring", LastNotify: make(map[string]time.Time), } // Test an OK response go func() { coordinator.evaluatorResponse <- responseOK // After a short wait, close the quit channel to release the responseLoop time.Sleep(100 * time.Millisecond) close(coordinator.quitChannel) }() coordinator.running.Add(1) coordinator.responseLoop() coordinator.running.Wait() close(coordinator.evaluatorResponse) time.Sleep(100 * time.Millisecond) group := coordinator.clusters["testcluster"].Groups["testgroup"] assert.Equalf(t, "", group.ID, "Expected group incident ID to be empty, not %v", group.ID) assert.True(t, group.Start.IsZero(), "Expected group incident start time to be cleared") assert.True(t, group.LastNotify["test"].IsZero(), "Expected group last time to be unset") module := coordinator.modules["test"].(*NullNotifier) assert.True(t, module.CalledAcceptConsumerGroup, "Expected module 'test' AcceptConsumerGroup to be called") } func TestCoordinator_responseLoop_NoIncidentError(t *testing.T) { coordinator := fixtureCoordinator() // We expect notifyModule to be called for the Null module in all cases (except NotFound) responseError := &protocol.ConsumerGroupStatus{ Cluster: "testcluster", Group: "testgroup", Status: protocol.StatusError, } coordinator.notifyModuleFunc = func(module Module, status *protocol.ConsumerGroupStatus, startTime time.Time, eventId string) { defer coordinator.running.Done() assert.Equal(t, "test", module.GetName(), "Expected to be called with the null notifier module") assert.Equal(t, responseError, status, "Expected to be called with responseError as the status") assert.False(t, startTime.IsZero(), "Expected to be called with a valid startTime, not zero") assert.NotEqual(t, "", eventId, "Expected to be called with a new eventId, not empty") } coordinator.Configure() // A test cluster and group to receive response for coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ LastNotify: make(map[string]time.Time), } // Test an Error response go func() { coordinator.evaluatorResponse <- responseError // After a short wait, close the quit channel to release the responseLoop time.Sleep(100 * time.Millisecond) close(coordinator.quitChannel) }() coordinator.running.Add(1) coordinator.responseLoop() coordinator.running.Wait() close(coordinator.evaluatorResponse) time.Sleep(100 * time.Millisecond) group := coordinator.clusters["testcluster"].Groups["testgroup"] assert.NotEqual(t, "", group.ID, "Expected group incident ID to be set, not empty") assert.False(t, group.Start.IsZero(), "Expected group incident start time to be set") assert.True(t, group.LastNotify["test"].IsZero(), "Expected group last time to be unset (as real notifyFunc was not called)") module := coordinator.modules["test"].(*NullNotifier) assert.True(t, module.CalledAcceptConsumerGroup, "Expected module 'test' AcceptConsumerGroup to be called") } func TestCoordinator_responseLoop_HaveIncidentError(t *testing.T) { coordinator := fixtureCoordinator() // We expect notifyModule to be called for the Null module in all cases (except NotFound) mockStartTime, _ := time.Parse(time.RFC3339, "2012-11-01T22:08:41+00:00") responseError := &protocol.ConsumerGroupStatus{ Cluster: "testcluster", Group: "testgroup", Status: protocol.StatusError, } coordinator.notifyModuleFunc = func(module Module, status *protocol.ConsumerGroupStatus, startTime time.Time, eventId string) { defer coordinator.running.Done() assert.Equal(t, "test", module.GetName(), "Expected to be called with the null notifier module") assert.Equal(t, responseError, status, "Expected to be called with responseError as the status") assert.Equal(t, mockStartTime, startTime, "Expected to be called with mockStartTime as the startTime") assert.Equalf(t, "testidstring", eventId, "Expected to be called with eventId as 'testidstring', not %v", eventId) } coordinator.Configure() // A test cluster and group to receive response for, with the incident active coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ Start: mockStartTime, ID: "testidstring", LastNotify: make(map[string]time.Time), } // Test an Error response go func() { coordinator.evaluatorResponse <- responseError // After a short wait, close the quit channel to release the responseLoop time.Sleep(100 * time.Millisecond) close(coordinator.quitChannel) }() coordinator.running.Add(1) coordinator.responseLoop() coordinator.running.Wait() close(coordinator.evaluatorResponse) time.Sleep(100 * time.Millisecond) group := coordinator.clusters["testcluster"].Groups["testgroup"] assert.Equal(t, mockStartTime, group.Start, "Expected group incident start time to be unchanged") assert.Equalf(t, "testidstring", group.ID, "Expected group incident ID to be 'testidstring', not %v", group.ID) assert.True(t, group.LastNotify["test"].IsZero(), "Expected group last time to be unset (as real notifyFunc was not called)") module := coordinator.modules["test"].(*NullNotifier) assert.True(t, module.CalledAcceptConsumerGroup, "Expected module 'test' AcceptConsumerGroup to be called") } var notifyModuleTests = []struct { Threshold int Status protocol.StatusConstant Existing bool SendClose bool ExpectSend bool ExpectClose bool ExpectID bool }{ /*{1, 0, false, false, false, false, false}, {2, 0, false, false, false, false, false}, {1, 0, true, false, false, false, false}, {1, 0, false, true, false, false, false}, {1, 0, true, true, false, false, false}, */ {1, 1, false, false, true, false, false}, {1, 1, false, true, true, false, false}, {1, 1, true, false, true, false, false}, {1, 1, true, true, true, true, false}, {1, 2, false, false, true, false, true}, {1, 2, false, true, true, false, true}, {1, 2, true, false, true, false, true}, {1, 2, true, true, true, false, true}, {3, 2, false, false, false, false, true}, {3, 2, false, true, false, false, true}, {3, 2, true, false, false, false, true}, {3, 2, true, true, false, false, true}, {2, 1, false, false, false, false, false}, {2, 1, false, true, false, false, false}, {2, 1, true, false, false, false, false}, {2, 1, true, true, true, true, false}, } func TestCoordinator_checkAndSendResponseToModules(t *testing.T) { // We don't need to configure the coordinator - just set up the data structures coordinator := fixtureCoordinator() coordinator.modules = make(map[string]protocol.Module) coordinator.clusters = make(map[string]*clusterGroups) coordinator.notifyModuleFunc = coordinator.notifyModule coordinator.clusterLock = &sync.RWMutex{} // A test cluster and group to send notification for (so the func can check/set last time) coordinator.clusters["testcluster"] = &clusterGroups{ Lock: &sync.RWMutex{}, Groups: make(map[string]*consumerGroup), } coordinator.clusters["testcluster"].Groups["testgroup"] = &consumerGroup{ LastNotify: make(map[string]time.Time), } mockStartTime, _ := time.Parse(time.RFC3339, "2012-11-01T22:08:41+00:00") for i, testSet := range notifyModuleTests { fmt.Printf("Running test %v - %v\n", i, testSet) // Set the module threshold and send-close configs viper.Reset() viper.Set("notifier.test.threshold", testSet.Threshold) viper.Set("notifier.test.send-close", testSet.SendClose) // Should there be an existing incident for this test? group := coordinator.clusters["testcluster"].Groups["testgroup"] delete(group.LastNotify, "test") if testSet.Existing { group.ID = "testidstring" group.Start = mockStartTime } else { group.ID = "" group.Start = time.Time{} } // Set up the response status to send // Sending i as the TotalPartitions is a hack to make mock errors a little easier to understand response := &protocol.ConsumerGroupStatus{ Cluster: "testcluster", Group: "testgroup", Status: testSet.Status, TotalPartitions: i, } // Set up the mock module and expected calls mockModule := &helpers.MockModule{} coordinator.modules["test"] = mockModule mockModule.On("GetName").Return("test") mockModule.On("GetGroupWhitelist").Return((*regexp.Regexp)(nil)) mockModule.On("GetGroupBlacklist").Return((*regexp.Regexp)(nil)) mockModule.On("AcceptConsumerGroup", response).Return(true) if testSet.ExpectSend { mockModule.On("Notify", response, mock.MatchedBy(func(s string) bool { return true }), mock.MatchedBy(func(t time.Time) bool { return true }), testSet.ExpectClose).Return() } // Call the func with a response that has the appropriate status coordinator.running.Add(1) coordinator.checkAndSendResponseToModules(response) mockModule.AssertExpectations(t) if testSet.ExpectSend && (!testSet.ExpectClose) { assert.Falsef(t, group.LastNotify["test"].IsZero(), "Test %v: Expected group last time to be set", i) } else { assert.Truef(t, group.LastNotify["test"].IsZero(), "Test %v: Expected group last time to remain unset", i) } // Check whether or not the incident is as expected afterwards if testSet.ExpectID { if testSet.Existing { assert.Equalf(t, "testidstring", group.ID, "Test %v: Expected group incident ID to be testidstring, not %v", i, group.ID) assert.Equalf(t, mockStartTime, group.Start, "Test %v: Expected group incident start time to be mock time, not %v", i, group.Start) } else { assert.NotEqualf(t, "", group.ID, "Test %v: Expected group incident ID to be not empty", i) assert.Falsef(t, group.Start.IsZero(), "Test %v: Expected group incident start time to be set", i) } } else { assert.Equalf(t, "", group.ID, "Test %v: Expected group incident ID to be empty, not %v", i, group.ID) assert.Truef(t, group.Start.IsZero(), "Test %v: Expected group incident start time to be unset", i) } } } func TestCoordinator_ExecuteTemplate(t *testing.T) { tmpl, _ := template.New("test").Parse("{{.ID}} {{.Cluster}} {{.Group}} {{.Result.Status}}") status := &protocol.ConsumerGroupStatus{ Status: protocol.StatusOK, Cluster: "testcluster", Group: "testgroup", } extras := make(map[string]string) extras["foo"] = "bar" bytesToSend, err := executeTemplate(tmpl, extras, status, "testidstring", time.Now()) assert.Nil(t, err, "Expected no error to be returned") assert.Equalf(t, "testidstring testcluster testgroup OK", bytesToSend.String(), "Unexpected, got: %v", bytesToSend.String()) } burrow-1.2.1/core/internal/notifier/email.go000066400000000000000000000133041343357346000210450ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "bytes" "errors" "fmt" "net/smtp" "regexp" "strings" "text/template" "time" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" "github.com/spf13/viper" ) // EmailNotifier is a module which can be used to send notifications of consumer group status via email messages. One // email is sent for each consumer group that matches the whitelist/blacklist and the status threshold. type EmailNotifier struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string groupWhitelist *regexp.Regexp groupBlacklist *regexp.Regexp extras map[string]string templateOpen *template.Template templateClose *template.Template from string to string auth smtp.Auth serverWithPort string sendMailFunc func(string, smtp.Auth, string, []string, []byte) error } // Configure validates the configuration of the email notifier. At minimum, there must be a valid server, port, from // address, and to address. If any of these are missing or incorrect, this func will panic with an explanatory message. // It is also possible to specify an auth-type of either "plain" or "crammd5", along with a username and password. func (module *EmailNotifier) Configure(name string, configRoot string) { module.name = name // Abstract the SendMail call so we can test if module.sendMailFunc == nil { module.sendMailFunc = smtp.SendMail } module.serverWithPort = fmt.Sprintf("%s:%v", viper.GetString(configRoot+".server"), viper.GetString(configRoot+".port")) if !helpers.ValidateHostList([]string{module.serverWithPort}) { module.Log.Panic("bad server or port") panic(errors.New("configuration error")) } module.from = viper.GetString(configRoot + ".from") if module.from == "" { module.Log.Panic("missing from address") panic(errors.New("configuration error")) } module.to = viper.GetString(configRoot + ".to") if module.to == "" { module.Log.Panic("missing to address") panic(errors.New("configuration error")) } // Set up SMTP authentication switch strings.ToLower(viper.GetString(configRoot + ".auth-type")) { case "plain": module.auth = smtp.PlainAuth("", viper.GetString(configRoot+".username"), viper.GetString(configRoot+".password"), viper.GetString(configRoot+".server")) case "crammd5": module.auth = smtp.CRAMMD5Auth(viper.GetString(configRoot+".username"), viper.GetString(configRoot+".password")) case "": module.auth = nil default: module.Log.Panic("unknown auth type") panic(errors.New("configuration error")) } } // Start is a no-op for the email notifier. It always returns no error func (module *EmailNotifier) Start() error { return nil } // Stop is a no-op for the email notifier. It always returns no error func (module *EmailNotifier) Stop() error { return nil } // GetName returns the configured name of this module func (module *EmailNotifier) GetName() string { return module.name } // GetGroupWhitelist returns the compiled group whitelist (or nil, if there is not one) func (module *EmailNotifier) GetGroupWhitelist() *regexp.Regexp { return module.groupWhitelist } // GetGroupBlacklist returns the compiled group blacklist (or nil, if there is not one) func (module *EmailNotifier) GetGroupBlacklist() *regexp.Regexp { return module.groupBlacklist } // GetLogger returns the configured zap.Logger for this notifier func (module *EmailNotifier) GetLogger() *zap.Logger { return module.Log } // AcceptConsumerGroup has no additional function for the email notifier, and so always returns true func (module *EmailNotifier) AcceptConsumerGroup(status *protocol.ConsumerGroupStatus) bool { return true } // Notify sends a single email message, with the from and to set to the configured addresses for the notifier. The // status, eventID, and startTime are all passed to the template for compiling the message. If stateGood is true, the // "close" template is used. Otherwise, the "open" template is used. func (module *EmailNotifier) Notify(status *protocol.ConsumerGroupStatus, eventID string, startTime time.Time, stateGood bool) { logger := module.Log.With( zap.String("cluster", status.Cluster), zap.String("group", status.Group), zap.String("id", eventID), zap.String("status", status.Status.String()), ) var tmpl *template.Template if stateGood { tmpl = module.templateClose } else { tmpl = module.templateOpen } // Put the from and to lines in without the template. Template should set the subject line, followed by a blank line bytesToSend := bytes.NewBufferString("From: " + module.from + "\nTo: " + module.to + "\n") messageBody, err := executeTemplate(tmpl, module.extras, status, eventID, startTime) if err != nil { logger.Error("failed to assemble", zap.Error(err)) return } bytesToSend.Write(messageBody.Bytes()) err = module.sendMailFunc(module.serverWithPort, module.auth, module.from, []string{module.to}, bytesToSend.Bytes()) if err != nil { logger.Error("failed to send", zap.Error(err)) } } burrow-1.2.1/core/internal/notifier/email_test.go000066400000000000000000000130601343357346000221030ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "net/smtp" "text/template" "time" "github.com/stretchr/testify/assert" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) func fixtureEmailNotifier() *EmailNotifier { module := EmailNotifier{ Log: zap.NewNop(), } module.App = &protocol.ApplicationContext{} viper.Reset() viper.Set("notifier.test.class-name", "email") viper.Set("notifier.test.template-open", "template_open") viper.Set("notifier.test.template-close", "template_close") viper.Set("notifier.test.send-close", false) viper.Set("notifier.test.server", "test.example.com") viper.Set("notifier.test.port", 587) viper.Set("notifier.test.from", "sender@example.com") viper.Set("notifier.test.to", "receiver@example.com") return &module } func TestEmailNotifier_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(EmailNotifier)) assert.Implements(t, (*Module)(nil), new(EmailNotifier)) } func TestEmailNotifier_Configure(t *testing.T) { module := fixtureEmailNotifier() module.Configure("test", "notifier.test") assert.Equalf(t, "test.example.com:587", module.serverWithPort, "Expected serverWithPort to be test.example.com:587, not %v", module.serverWithPort) assert.NotNil(t, module.sendMailFunc, "Expected sendMailFunc to get set to smtp.SendMail") assert.Nil(t, module.auth, "Expected auth to be set to nil") } func TestEmailNotifier_Configure_BasicAuth(t *testing.T) { module := fixtureEmailNotifier() viper.Set("notifier.test.auth-type", "plain") viper.Set("notifier.test.username", "user") viper.Set("notifier.test.password", "pass") module.Configure("test", "notifier.test") assert.NotNil(t, module.auth, "Expected auth to be set") } func TestEmailNotifier_Configure_CramMD5(t *testing.T) { module := fixtureEmailNotifier() viper.Set("notifier.test.auth-type", "CramMD5") viper.Set("notifier.test.username", "user") viper.Set("notifier.test.password", "pass") module.Configure("test", "notifier.test") assert.NotNil(t, module.auth, "Expected auth to be set") } func TestEmailNotifier_StartStop(t *testing.T) { module := fixtureEmailNotifier() module.Configure("test", "notifier.test") err := module.Start() assert.Nil(t, err, "Expected Start to return no error") err = module.Stop() assert.Nil(t, err, "Expected Stop to return no error") } func TestEmailNotifier_AcceptConsumerGroup(t *testing.T) { module := fixtureEmailNotifier() module.Configure("test", "notifier.test") // Should always return true assert.True(t, module.AcceptConsumerGroup(&protocol.ConsumerGroupStatus{}), "Expected any status to return True") } func TestEmailNotifier_Notify_Open(t *testing.T) { module := fixtureEmailNotifier() viper.Set("notifier.test.auth-type", "plain") viper.Set("notifier.test.username", "user") viper.Set("notifier.test.password", "pass") module.sendMailFunc = func(server string, auth smtp.Auth, from string, to []string, bytesToSend []byte) error { assert.Equalf(t, "test.example.com:587", server, "Expected server to be test.example.com:587, not %v", server) assert.NotNil(t, auth, "Expected auth to not be nil") assert.Equalf(t, "sender@example.com", from, "Expected from to be sender@example.com, not %v", from) assert.Lenf(t, to, 1, "Expected one to address, not %v", len(to)) assert.Equalf(t, "receiver@example.com", to[0], "Expected to to be receiver@example.com, not %v", to[0]) assert.Equalf(t, []byte("From: sender@example.com\nTo: receiver@example.com\ntestidstring testcluster testgroup WARN"), bytesToSend, "Unexpected bytes, got: %v", bytesToSend) return nil } // Template for testing module.templateOpen, _ = template.New("test").Parse("{{.Id}} {{.Cluster}} {{.Group}} {{.Result.Status}}") module.Configure("test", "notifier.test") status := &protocol.ConsumerGroupStatus{ Status: protocol.StatusWarning, Cluster: "testcluster", Group: "testgroup", } module.Notify(status, "testidstring", time.Now(), false) } func TestEmailNotifier_Notify_Close(t *testing.T) { module := fixtureEmailNotifier() module.sendMailFunc = func(server string, auth smtp.Auth, from string, to []string, bytesToSend []byte) error { assert.Equalf(t, "test.example.com:587", server, "Expected server to be test.example.com:587, not %v", server) assert.Nil(t, auth, "Expected auth to be nil") assert.Equalf(t, "sender@example.com", from, "Expected from to be sender@example.com, not %v", from) assert.Lenf(t, to, 1, "Expected one to address, not %v", len(to)) assert.Equalf(t, "receiver@example.com", to[0], "Expected to to be receiver@example.com, not %v", to[0]) assert.Equalf(t, []byte("From: sender@example.com\nTo: receiver@example.com\ntestidstring testcluster testgroup OK"), bytesToSend, "Unexpected bytes, got: %v", bytesToSend) return nil } // Template for testing module.templateClose, _ = template.New("test").Parse("{{.Id}} {{.Cluster}} {{.Group}} {{.Result.Status}}") module.Configure("test", "notifier.test") status := &protocol.ConsumerGroupStatus{ Status: protocol.StatusOK, Cluster: "testcluster", Group: "testgroup", } module.Notify(status, "testidstring", time.Now(), true) } burrow-1.2.1/core/internal/notifier/helpers.go000066400000000000000000000073641343357346000214310ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "encoding/json" "text/template" "time" "bytes" "github.com/linkedin/Burrow/core/protocol" ) // executeTemplate provides a common interface for notifier modules to call to process a text/template in the context // of a protocol.ConsumerGroupStatus and create a message to use in a notification. func executeTemplate(tmpl *template.Template, extras map[string]string, status *protocol.ConsumerGroupStatus, eventID string, startTime time.Time) (*bytes.Buffer, error) { bytesToSend := new(bytes.Buffer) err := tmpl.Execute(bytesToSend, struct { Cluster string Group string ID string Start time.Time Extras map[string]string Result protocol.ConsumerGroupStatus }{ Cluster: status.Cluster, Group: status.Group, ID: eventID, Start: startTime, Extras: extras, Result: *status, }) if err != nil { return nil, err } return bytesToSend, nil } // Helper functions for templates var helperFunctionMap = template.FuncMap{ "jsonencoder": templateJSONEncoder, "topicsbystatus": classifyTopicsByStatus, "partitioncounts": templateCountPartitions, "add": templateAdd, "minus": templateMinus, "multiply": templateMultiply, "divide": templateDivide, "maxlag": maxLagHelper, "formattimestamp": formatTimestamp, } // Helper function for the templates to encode an object into a JSON string func templateJSONEncoder(encodeMe interface{}) string { jsonStr, _ := json.Marshal(encodeMe) return string(jsonStr) } // Helper - recategorize partitions as a map of lists // map[string][]string => status short name -> list of topics func classifyTopicsByStatus(partitions []*protocol.PartitionStatus) map[string][]string { tmpMap := make(map[string]map[string]bool) for _, partition := range partitions { if _, ok := tmpMap[partition.Status.String()]; !ok { tmpMap[partition.Status.String()] = make(map[string]bool) } tmpMap[partition.Status.String()][partition.Topic] = true } rv := make(map[string][]string) for status, topicMap := range tmpMap { rv[status] = make([]string, 0, len(topicMap)) for topic := range topicMap { rv[status] = append(rv[status], topic) } } return rv } // Template Helper - Return a map of partition counts // keys are warn, stop, stall, rewind, unknown func templateCountPartitions(partitions []*protocol.PartitionStatus) map[string]int { rv := map[string]int{ "warn": 0, "stop": 0, "stall": 0, "rewind": 0, "unknown": 0, } for _, partition := range partitions { switch partition.Status { case protocol.StatusOK: case protocol.StatusWarning: rv["warn"]++ case protocol.StatusStop: rv["stop"]++ case protocol.StatusStall: rv["stall"]++ case protocol.StatusRewind: rv["rewind"]++ default: rv["unknown"]++ } } return rv } // Template Helper - do maths func templateAdd(a, b int) int { return a + b } func templateMinus(a, b int) int { return a - b } func templateMultiply(a, b int) int { return a * b } func templateDivide(a, b int) int { return a / b } func maxLagHelper(a *protocol.PartitionStatus) uint64 { if a == nil { return 0 } return a.CurrentLag } func formatTimestamp(timestamp int64, formatString string) string { return time.Unix(0, timestamp*int64(time.Millisecond)).Format(formatString) } burrow-1.2.1/core/internal/notifier/http.go000066400000000000000000000147121343357346000207410ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "errors" "io" "io/ioutil" "net" "net/http" "regexp" "text/template" "time" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) // HTTPNotifier is a module which can be used to send notifications of consumer group status via outbound HTTP calls to // another server. This is useful for informing another system, such as an alert system, when there is a problem. One // HTTP call is made for each consumer group that matches the whitelist/blacklist and the status threshold (though // keepalive connections will be used if configured). type HTTPNotifier struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string groupWhitelist *regexp.Regexp groupBlacklist *regexp.Regexp extras map[string]string urlOpen string urlClose string methodOpen string methodClose string templateOpen *template.Template templateClose *template.Template sendClose bool httpClient *http.Client } // Configure validates the configuration of the http notifier. At minimum, there must be a url-open specified, and if // send-close is set to true there must also be a url-close. If these are missing or incorrect, this func will panic // with an explanatory message. It is also possible to configure a specific method (such as POST or DELETE) to be used // with these URLs, as well as a timeout and keepalive for the HTTP client. func (module *HTTPNotifier) Configure(name string, configRoot string) { module.name = name // Validate and set defaults for profile configs module.urlOpen = viper.GetString(configRoot + ".url-open") if module.urlOpen == "" { module.Log.Panic("no url-open specified") panic(errors.New("configuration error")) } viper.SetDefault(configRoot+".method-open", "POST") module.methodOpen = viper.GetString(configRoot + ".method-open") module.sendClose = viper.GetBool(configRoot + ".send-close") if module.sendClose { module.urlClose = viper.GetString(configRoot + ".url-close") if module.urlClose == "" { module.Log.Panic("no url-close specified") panic(errors.New("configuration error")) } viper.SetDefault(configRoot+".method-close", "POST") module.methodClose = viper.GetString(configRoot + ".method-close") } // Set defaults for module-specific configs if needed viper.SetDefault(configRoot+".timeout", 5) viper.SetDefault(configRoot+".keepalive", 300) // Set up HTTP client module.httpClient = &http.Client{ Timeout: viper.GetDuration(configRoot+".timeout") * time.Second, Transport: &http.Transport{ Dial: (&net.Dialer{ KeepAlive: viper.GetDuration(configRoot+".keepalive") * time.Second, }).Dial, Proxy: http.ProxyFromEnvironment, }, } } // Start is a no-op for the http notifier. It always returns no error func (module *HTTPNotifier) Start() error { return nil } // Stop is a no-op for the http notifier. It always returns no error func (module *HTTPNotifier) Stop() error { return nil } // GetName returns the configured name of this module func (module *HTTPNotifier) GetName() string { return module.name } // GetGroupWhitelist returns the compiled group whitelist (or nil, if there is not one) func (module *HTTPNotifier) GetGroupWhitelist() *regexp.Regexp { return module.groupWhitelist } // GetGroupBlacklist returns the compiled group blacklist (or nil, if there is not one) func (module *HTTPNotifier) GetGroupBlacklist() *regexp.Regexp { return module.groupBlacklist } // GetLogger returns the configured zap.Logger for this notifier func (module *HTTPNotifier) GetLogger() *zap.Logger { return module.Log } // AcceptConsumerGroup has no additional function for the http notifier, and so always returns true func (module *HTTPNotifier) AcceptConsumerGroup(status *protocol.ConsumerGroupStatus) bool { return true } // Notify makes a single outbound HTTP request. The status, eventID, and startTime are all passed to the template for // compiling the request body. If stateGood is true, the "close" template and URL are used. Otherwise, the "open" // template and URL are used. func (module *HTTPNotifier) Notify(status *protocol.ConsumerGroupStatus, eventID string, startTime time.Time, stateGood bool) { logger := module.Log.With( zap.String("cluster", status.Cluster), zap.String("group", status.Group), zap.String("id", eventID), zap.String("status", status.Status.String()), ) var tmpl *template.Template var method string var url string if stateGood { tmpl = module.templateClose method = module.methodClose url = module.urlClose } else { tmpl = module.templateOpen method = module.methodOpen url = module.urlOpen } bytesToSend, err := executeTemplate(tmpl, module.extras, status, eventID, startTime) if err != nil { logger.Error("failed to assemble message", zap.Error(err)) return } // Send request to HTTP endpoint req, err := http.NewRequest(method, url, bytesToSend) if err != nil { logger.Error("failed to create request", zap.Error(err)) return } username := viper.GetString("notifier." + module.name + ".username") if username != "" { // Add basic auth using the provided username and password req.SetBasicAuth(viper.GetString("notifier."+module.name+".username"), viper.GetString("notifier."+module.name+".password")) } req.Header.Set("Content-Type", "application/json") for header, value := range viper.GetStringMapString("notifier." + module.name + ".headers") { req.Header.Set(header, value) } resp, err := module.httpClient.Do(req) if err != nil { logger.Error("failed to send", zap.Error(err)) return } io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() if (resp.StatusCode >= 200) && (resp.StatusCode <= 299) { logger.Debug("sent") } else { logger.Error("failed to send", zap.Int("response", resp.StatusCode)) } } burrow-1.2.1/core/internal/notifier/http_test.go000066400000000000000000000150441343357346000217770ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "encoding/json" "fmt" "net/http" "text/template" "time" "github.com/stretchr/testify/assert" "net/http/httptest" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) func fixtureHTTPNotifier() *HTTPNotifier { module := HTTPNotifier{ Log: zap.NewNop(), } module.App = &protocol.ApplicationContext{} viper.Reset() viper.Set("notifier.test.class-name", "http") viper.Set("notifier.test.url-open", "url_open") viper.Set("notifier.test.url-close", "url_close") viper.Set("notifier.test.template-open", "template_open") viper.Set("notifier.test.template-close", "template_close") viper.Set("notifier.test.send-close", false) viper.Set("notifier.test.headers", map[string]string{"Token": "testtoken"}) return &module } func TestHttpNotifier_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(HTTPNotifier)) assert.Implements(t, (*Module)(nil), new(HTTPNotifier)) } func TestHttpNotifier_Configure(t *testing.T) { module := fixtureHTTPNotifier() module.Configure("test", "notifier.test") assert.NotNil(t, module.httpClient, "Expected httpClient to be set with a client object") } func TestHttpNotifier_StartStop(t *testing.T) { module := fixtureHTTPNotifier() module.Configure("test", "notifier.test") err := module.Start() assert.Nil(t, err, "Expected Start to return no error") err = module.Stop() assert.Nil(t, err, "Expected Stop to return no error") } func TestHttpNotifier_AcceptConsumerGroup(t *testing.T) { module := fixtureHTTPNotifier() module.Configure("test", "notifier.test") // Should always return true assert.True(t, module.AcceptConsumerGroup(&protocol.ConsumerGroupStatus{}), "Expected any status to return True") } // Struct that will be used for sending HTTP requests for testing type HTTPRequest struct { Template string ID string Cluster string Group string } func TestHttpNotifier_Notify_Open(t *testing.T) { // handler that validates that we get the right values requestHandler := func(w http.ResponseWriter, r *http.Request) { // Must get an appropriate Content-Type header headers, ok := r.Header["Content-Type"] assert.True(t, ok, "Expected to receive Content-Type header") assert.Len(t, headers, 1, "Expected to receive exactly one Content-Type header") assert.Equalf(t, "application/json", headers[0], "Expected Content-Type header to be 'application/json', not '%v'", headers[0]) tokenHeaders, ok := r.Header["Token"] assert.True(t, ok, "Expected to receive Token header") assert.Equalf(t, "testtoken", tokenHeaders[0], "Expected Token header to be 'testtoken', not '%v'", tokenHeaders[0]) decoder := json.NewDecoder(r.Body) var req HTTPRequest err := decoder.Decode(&req) if err != nil { assert.Failf(t, "Failed to decode message body", "Failed to decode message body: %v", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } assert.Equalf(t, "template_open", req.Template, "Expected Template to be template_open, not %v", req.Template) assert.Equalf(t, "testidstring", req.ID, "Expected ID to be testidstring, not %v", req.ID) assert.Equalf(t, "testcluster", req.Cluster, "Expected Cluster to be testcluster, not %v", req.Cluster) assert.Equalf(t, "testgroup", req.Group, "Expected Group to be testgroup, not %v", req.Group) fmt.Fprint(w, "ok") } // create test server with handler ts := httptest.NewServer(http.HandlerFunc(requestHandler)) defer ts.Close() module := fixtureHTTPNotifier() viper.Set("notifier.test.url-open", ts.URL) // Template sends the ID, cluster, and group module.templateOpen, _ = template.New("test").Parse("{\"template\":\"template_open\",\"id\":\"{{.ID}}\",\"cluster\":\"{{.Cluster}}\",\"group\":\"{{.Group}}\"}") module.Configure("test", "notifier.test") status := &protocol.ConsumerGroupStatus{ Status: protocol.StatusWarning, Cluster: "testcluster", Group: "testgroup", } module.Notify(status, "testidstring", time.Now(), false) } func TestHttpNotifier_Notify_Close(t *testing.T) { // handler that validates that we get the right values requestHandler := func(w http.ResponseWriter, r *http.Request) { // Must get an appropriate Content-Type header headers, ok := r.Header["Content-Type"] assert.True(t, ok, "Expected to receive Content-Type header") assert.Len(t, headers, 1, "Expected to receive exactly one Content-Type header") assert.Equalf(t, "application/json", headers[0], "Expected Content-Type header to be 'application/json', not '%v'", headers[0]) tokenHeaders, ok := r.Header["Token"] assert.True(t, ok, "Expected to receive Token header") assert.Equalf(t, "testtoken", tokenHeaders[0], "Expected Token header to be 'testtoken', not '%v'", tokenHeaders[0]) decoder := json.NewDecoder(r.Body) var req HTTPRequest err := decoder.Decode(&req) if err != nil { assert.Failf(t, "Failed to decode message body", "Failed to decode message body: %v", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } assert.Equalf(t, "template_close", req.Template, "Expected Template to be template_close, not %v", req.Template) assert.Equalf(t, "testidstring", req.ID, "Expected ID to be testidstring, not %v", req.ID) assert.Equalf(t, "testcluster", req.Cluster, "Expected Cluster to be testcluster, not %v", req.Cluster) assert.Equalf(t, "testgroup", req.Group, "Expected Group to be testgroup, not %v", req.Group) fmt.Fprint(w, "ok") } // create test server with handler ts := httptest.NewServer(http.HandlerFunc(requestHandler)) defer ts.Close() module := fixtureHTTPNotifier() viper.Set("notifier.test.send-close", true) viper.Set("notifier.test.url-close", ts.URL) // Template sends the ID, cluster, and group module.templateClose, _ = template.New("test").Parse("{\"template\":\"template_close\",\"id\":\"{{.ID}}\",\"cluster\":\"{{.Cluster}}\",\"group\":\"{{.Group}}\"}") module.Configure("test", "notifier.test") status := &protocol.ConsumerGroupStatus{ Status: protocol.StatusWarning, Cluster: "testcluster", Group: "testgroup", } module.Notify(status, "testidstring", time.Now(), true) } burrow-1.2.1/core/internal/notifier/null.go000066400000000000000000000065071343357346000207370ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package notifier import ( "regexp" "text/template" "time" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) // NullNotifier is a no-op notifier that can be used for testing purposes in place of a mock. It does not make any // external calls, and will record if specific funcs are called. type NullNotifier struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string groupWhitelist *regexp.Regexp groupBlacklist *regexp.Regexp extras map[string]string templateOpen *template.Template templateClose *template.Template // CalledConfigure is set to true if the Configure method is called CalledConfigure bool // CalledStart is set to true if the Start method is called CalledStart bool // CalledStop is set to true if the Stop method is called CalledStop bool // CalledNotify is set to true if the Notify method is called CalledNotify bool // CalledAcceptConsumerGroup is set to true if the AcceptConsumerGroup method is called CalledAcceptConsumerGroup bool } // Configure sets the module name, but performs no other functions for the null notifier func (module *NullNotifier) Configure(name string, configRoot string) { module.name = name module.CalledConfigure = true } // Start is a no-op for the null notifier. It always returns no error func (module *NullNotifier) Start() error { module.CalledStart = true return nil } // Stop is a no-op for the null notifier. It always returns no error func (module *NullNotifier) Stop() error { module.CalledStop = true return nil } // GetName returns the configured name of this module func (module *NullNotifier) GetName() string { return module.name } // GetGroupWhitelist returns the compiled group whitelist (or nil, if there is not one) func (module *NullNotifier) GetGroupWhitelist() *regexp.Regexp { return module.groupWhitelist } // GetGroupBlacklist returns the compiled group blacklist (or nil, if there is not one) func (module *NullNotifier) GetGroupBlacklist() *regexp.Regexp { return module.groupBlacklist } // GetLogger returns the configured zap.Logger for this notifier func (module *NullNotifier) GetLogger() *zap.Logger { return module.Log } // AcceptConsumerGroup has no additional function for the null notifier, and so always returns true func (module *NullNotifier) AcceptConsumerGroup(status *protocol.ConsumerGroupStatus) bool { module.CalledAcceptConsumerGroup = true return true } // Notify is a no-op for the null notifier func (module *NullNotifier) Notify(status *protocol.ConsumerGroupStatus, eventID string, startTime time.Time, stateGood bool) { module.CalledNotify = true } burrow-1.2.1/core/internal/storage/000077500000000000000000000000001343357346000172535ustar00rootroot00000000000000burrow-1.2.1/core/internal/storage/coordinator.go000066400000000000000000000142651343357346000221350ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package storage - Data storage subsystem. // The storage subsystem receives information from the cluster and consumer subsystems and serves that information out // to other subsystems on request. // // Modules // // Currently, only one module is provided: // // * inmemory - Store all information in a set of in-memory maps package storage import ( "errors" "sync" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // Module (storage) is responsible for maintaining all the broker and consumer offsets for all clusters that Burrow // watches. It must accept and respond to all protocol.StorageRequest types. This interface conforms to the overall // protocol.Module interface, but it adds a func to fetch the channel that the module is listening on for requests, so // that requests can be forwarded to it by the coordinator. type Module interface { protocol.Module GetCommunicationChannel() chan *protocol.StorageRequest } // Coordinator (storage) manages a single storage module (only one module is supported at this time), making sure it // is configured, started, and stopped at the appropriate time. It is also responsible for listening to the // StorageChannel that is provided in the application context and forwarding those requests to the storage module. If // no storage module has been configured explicitly, the coordinator starts the inmemory module as a default. type Coordinator struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger quitChannel chan struct{} modules map[string]protocol.Module running sync.WaitGroup } // getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there // is any error, it will panic with an appropriate message describing the problem. func getModuleForClass(app *protocol.ApplicationContext, moduleName string, className string) Module { switch className { case "inmemory": return &InMemoryStorage{ App: app, Log: app.Logger.With( zap.String("type", "module"), zap.String("coordinator", "storage"), zap.String("class", className), zap.String("name", moduleName), ), } default: panic("Unknown storage className provided: " + className) } } // Configure is called to create the configured storage module and call its Configure func to validate the // configuration and set it up. The coordinator will panic is more than one module is configured, and if no modules have // been configured, it will set up a default inmemory storage module. If there are any problems, it is expected that // this func will panic with a descriptive error message, as configuration failures are not recoverable errors. func (sc *Coordinator) Configure() { sc.Log.Info("configuring") sc.quitChannel = make(chan struct{}) sc.modules = make(map[string]protocol.Module) sc.running = sync.WaitGroup{} modules := viper.GetStringMap("storage") switch len(modules) { case 0: // Create a default module viper.Set("storage.default.class-name", "inmemory") modules = viper.GetStringMap("storage") case 1: // Have one module. Just continue break default: panic("Only one storage module must be configured") } // Create all configured storage modules, add to list of storage for name := range modules { configRoot := "storage." + name module := getModuleForClass(sc.App, name, viper.GetString(configRoot+".class-name")) module.Configure(name, configRoot) sc.modules[name] = module } } // Start calls the storage module's underlying Start func. If the module Start returns an error, this func stops // immediately and returns that error to the caller. // // We also start a request forwarder goroutine. This listens to the StorageChannel that is provided in the application // context that all modules receive, and forwards those requests to the storage modules. At the present time, the // storage subsystem only supports one module, so this is a simple "accept and forward". func (sc *Coordinator) Start() error { sc.Log.Info("starting") // Start Storage modules err := helpers.StartCoordinatorModules(sc.modules) if err != nil { return errors.New("Error starting storage module: " + err.Error()) } // Start request forwarder go sc.mainLoop() return nil } // Stop calls the configured storage module's underlying Stop func. It is expected that the module Stop will not return // until the module has been completely stopped. While an error can be returned, this func always returns no error, as // a failure during stopping is not a critical failure func (sc *Coordinator) Stop() error { sc.Log.Info("stopping") close(sc.quitChannel) sc.running.Wait() // The individual storage modules can choose whether or not to implement a wait in the Stop routine helpers.StopCoordinatorModules(sc.modules) return nil } func (sc *Coordinator) mainLoop() { sc.running.Add(1) defer sc.running.Done() // We only support 1 module right now, so only send to that module var channel chan *protocol.StorageRequest for _, module := range sc.modules { channel = module.(Module).GetCommunicationChannel() } for { select { case request := <-sc.App.StorageChannel: // Yes, this forwarder is silly. However, in the future we want to support multiple storage modules // concurrently. However, that will require implementing a router that properly handles sets and // fetches and makes sure only 1 module responds to fetches channel <- request case <-sc.quitChannel: return } } } burrow-1.2.1/core/internal/storage/coordinator_test.go000066400000000000000000000054151343357346000231710ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package storage import ( "github.com/spf13/viper" "go.uber.org/zap" "github.com/stretchr/testify/assert" "testing" "github.com/linkedin/Burrow/core/protocol" "time" ) func fixtureCoordinator() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("storage.test.class-name", "inmemory") viper.Set("cluster.testcluster.class-name", "kafka") viper.Set("cluster.testcluster.servers", []string{"broker1.example.com:1234"}) return &coordinator } func TestCoordinator_ImplementsCoordinator(t *testing.T) { assert.Implements(t, (*protocol.Coordinator)(nil), new(Coordinator)) } func TestCoordinator_Configure(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) } func TestCoordinator_Configure_NoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Reset() coordinator.Configure() assert.Lenf(t, coordinator.modules, 1, "Expected 1 module configured, not %v", len(coordinator.modules)) } func TestCoordinator_Configure_TwoModules(t *testing.T) { coordinator := fixtureCoordinator() viper.Set("storage.anothertest.class-name", "inmemory") assert.Panics(t, coordinator.Configure, "Expected panic") } func TestCoordinator_Start(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() coordinator.Start() // Best is to test a request that we know the response to request := &protocol.StorageRequest{ RequestType: protocol.StorageFetchClusters, Reply: make(chan interface{}), } coordinator.App.StorageChannel <- request response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 1, "One entry not returned") assert.Equalf(t, val[0], "testcluster", "Expected return value to be 'testcluster', not %v", val[0]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") time.Sleep(10 * time.Millisecond) coordinator.Stop() } func TestCoordinator_MultipleRequests(t *testing.T) { coordinator := CoordinatorWithOffsets() coordinator.Stop() } burrow-1.2.1/core/internal/storage/fixtures.go000066400000000000000000000057071343357346000214640ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package storage import ( "time" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) // CoordinatorWithOffsets sets up a Coordinator with a single inmemory module defined. This module is loaded with // offsets for a test cluster and group. This func should never be called in normal code. It is only provided to // facilitate testing by other subsystems. func CoordinatorWithOffsets() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("storage.test.class-name", "inmemory") viper.Set("storage.test.intervals", 10) viper.Set("storage.test.min-distance", 0) viper.Set("storage.test.group-whitelist", "") viper.Set("cluster.testcluster.class-name", "kafka") coordinator.Configure() coordinator.Start() // Add a broker offset coordinator.App.StorageChannel <- &protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "testcluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 1, Offset: 4321, Timestamp: 9876, } time.Sleep(100 * time.Millisecond) // Add consumer offsets for a full ring startTime := (time.Now().Unix() * 1000) - 100000 for i := 0; i < 10; i++ { coordinator.App.StorageChannel <- &protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Offset: int64(1000 + (i * 100)), Timestamp: startTime + int64((i * 10000)), } // If we don't sleep while submitting these, we can end up with false test results due to race conditions time.Sleep(10 * time.Millisecond) } // Add a second group with a partial ring for i := 0; i < 5; i++ { coordinator.App.StorageChannel <- &protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup2", Partition: 0, Offset: int64(1000 + (i * 100)), Timestamp: startTime + int64((i * 10000)), } // If we don't sleep while submitting these, we can end up with false test results due to race conditions time.Sleep(10 * time.Millisecond) } // Sleep just a little more to make sure everything's processed time.Sleep(100 * time.Millisecond) return &coordinator } burrow-1.2.1/core/internal/storage/inmemory.go000066400000000000000000000647621343357346000214600ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package storage import ( "container/ring" "math/rand" "regexp" "sync" "time" "github.com/OneOfOne/xxhash" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) // InMemoryStorage is a storage module that maintains the entire data set in memory in a series of maps. It has a // configurable number of worker goroutines to service requests, and for requests that are group-specific, the group // and cluster name are used to hash the request to a consistent worker. This assures that requests for a group are // processed in order. type InMemoryStorage struct { // App is a pointer to the application context. This stores the channel to the storage subsystem App *protocol.ApplicationContext // Log is a logger that has been configured for this module to use. Normally, this means it has been set up with // fields that are appropriate to identify this coordinator Log *zap.Logger name string intervals int numWorkers int expireGroup int64 minDistance int64 queueDepth int requestChannel chan *protocol.StorageRequest workersRunning sync.WaitGroup mainRunning sync.WaitGroup offsets map[string]clusterOffsets groupWhitelist *regexp.Regexp groupBlacklist *regexp.Regexp workers []chan *protocol.StorageRequest } type brokerOffset struct { Offset int64 Timestamp int64 } type consumerPartition struct { offsets *ring.Ring owner string clientID string } type consumerGroup struct { // This lock is held when using the individual group, either for read or write lock *sync.RWMutex topics map[string][]*consumerPartition lastCommit int64 } type clusterOffsets struct { broker map[string][]*ring.Ring consumer map[string]*consumerGroup // This lock is used when modifying broker topics or offsets brokerLock *sync.RWMutex // This lock is used when modifying the overall consumer list // It does not need to be held for modifying an individual group consumerLock *sync.RWMutex } // Configure validates the configuration for the module, creates a channel to receive requests on, and sets up the // storage map. If no expiration time for groups is set, a default value of 7 days is used. If no interval count is // set, a default of 10 intervals is used. If no worker count is set, a default of 20 workers is used. func (module *InMemoryStorage) Configure(name string, configRoot string) { module.Log.Info("configuring") module.name = name // Set defaults for configs if needed viper.SetDefault(configRoot+".intervals", 10) viper.SetDefault(configRoot+".expire-group", 604800) viper.SetDefault(configRoot+".workers", 20) viper.SetDefault(configRoot+".queue-depth", 1) module.intervals = viper.GetInt(configRoot + ".intervals") module.expireGroup = viper.GetInt64(configRoot + ".expire-group") module.numWorkers = viper.GetInt(configRoot + ".workers") module.minDistance = viper.GetInt64(configRoot + ".min-distance") module.queueDepth = viper.GetInt(configRoot + ".queue-depth") module.requestChannel = make(chan *protocol.StorageRequest, module.queueDepth) module.workersRunning = sync.WaitGroup{} module.mainRunning = sync.WaitGroup{} module.offsets = make(map[string]clusterOffsets) whitelist := viper.GetString(configRoot + ".group-whitelist") if whitelist != "" { re, err := regexp.Compile(whitelist) if err != nil { module.Log.Panic("Failed to compile group whitelist") panic(err) } module.groupWhitelist = re } blacklist := viper.GetString(configRoot + ".group-blacklist") if blacklist != "" { re, err := regexp.Compile(blacklist) if err != nil { module.Log.Panic("Failed to compile group blacklist") panic(err) } module.groupBlacklist = re } } // GetCommunicationChannel returns the RequestChannel that has been setup for this module. func (module *InMemoryStorage) GetCommunicationChannel() chan *protocol.StorageRequest { return module.requestChannel } // Start sets up the rest of the storage map for each configured cluster. It then starts the configured number of // worker routines to handle requests. Finally, it starts a main loop which will receive requests and hash them to the // correct worker. func (module *InMemoryStorage) Start() error { module.Log.Info("starting") for cluster := range viper.GetStringMap("cluster") { module. offsets[cluster] = clusterOffsets{ broker: make(map[string][]*ring.Ring), consumer: make(map[string]*consumerGroup), brokerLock: &sync.RWMutex{}, consumerLock: &sync.RWMutex{}, } } // Start the appropriate number of workers, with a channel for each module.workers = make([]chan *protocol.StorageRequest, module.numWorkers) for i := 0; i < module.numWorkers; i++ { module.workers[i] = make(chan *protocol.StorageRequest, module.queueDepth) module.workersRunning.Add(1) go module.requestWorker(i, module.workers[i]) } module.mainRunning.Add(1) go module.mainLoop() return nil } // Stop closes the incoming request channel, which will close the main loop. It then closes each of the worker // channels, to close the workers, and waits for all goroutines to exit before returning. func (module *InMemoryStorage) Stop() error { module.Log.Info("stopping") close(module.requestChannel) module.mainRunning.Wait() for i := 0; i < module.numWorkers; i++ { close(module.workers[i]) } module.workersRunning.Wait() return nil } func (module *InMemoryStorage) requestWorker(workerNum int, requestChannel chan *protocol.StorageRequest) { defer module.workersRunning.Done() // Using a map for the request types avoids a bit of complexity below var requestTypeMap = map[protocol.StorageRequestConstant]func(*protocol.StorageRequest, *zap.Logger){ protocol.StorageSetBrokerOffset: module.addBrokerOffset, protocol.StorageSetConsumerOffset: module.addConsumerOffset, protocol.StorageSetConsumerOwner: module.addConsumerOwner, protocol.StorageSetDeleteTopic: module.deleteTopic, protocol.StorageSetDeleteGroup: module.deleteGroup, protocol.StorageFetchClusters: module.fetchClusterList, protocol.StorageFetchConsumers: module.fetchConsumerList, protocol.StorageFetchTopics: module.fetchTopicList, protocol.StorageFetchConsumer: module.fetchConsumer, protocol.StorageFetchTopic: module.fetchTopic, protocol.StorageClearConsumerOwners: module.clearConsumerOwners, protocol.StorageFetchConsumersForTopic: module.fetchConsumersForTopicList, } workerLogger := module.Log.With(zap.Int("worker", workerNum)) for r := range requestChannel { if requestFunc, ok := requestTypeMap[r.RequestType]; ok { requestFunc(r, workerLogger.With( zap.String("cluster", r.Cluster), zap.String("consumer", r.Group), zap.String("topic", r.Topic), zap.Int32("partition", r.Partition), zap.Int32("topic_partition_count", r.TopicPartitionCount), zap.Int64("offset", r.Offset), zap.Int64("timestamp", r.Timestamp), zap.String("owner", r.Owner), zap.String("client_id", r.ClientID), zap.String("request", r.RequestType.String()))) } } } func (module *InMemoryStorage) mainLoop() { defer module.mainRunning.Done() for r := range module.requestChannel { switch r.RequestType { case protocol.StorageSetBrokerOffset, protocol.StorageSetDeleteTopic, protocol.StorageFetchClusters, protocol.StorageFetchConsumers, protocol.StorageFetchTopics, protocol.StorageFetchTopic, protocol.StorageFetchConsumersForTopic: // Send to any worker module.workers[int(rand.Int31n(int32(module.numWorkers)))] <- r case protocol.StorageSetConsumerOffset, protocol.StorageSetConsumerOwner, protocol.StorageSetDeleteGroup, protocol.StorageClearConsumerOwners, protocol.StorageFetchConsumer: // Hash to a consistent worker module.workers[int(xxhash.ChecksumString64(r.Cluster+r.Group)%uint64(module.numWorkers))] <- r default: module.Log.Error("unknown storage request type", zap.Int("request_type", int(r.RequestType)), ) if r.Reply != nil { close(r.Reply) } } } } func (module *InMemoryStorage) addBrokerOffset(request *protocol.StorageRequest, requestLogger *zap.Logger) { clusterMap, ok := module.offsets[request.Cluster] if !ok { // Ignore offsets for clusters that we don't know about - should never happen anyways requestLogger.Warn("unknown cluster") return } clusterMap.brokerLock.Lock() defer clusterMap.brokerLock.Unlock() topicList, ok := clusterMap.broker[request.Topic] if !ok { clusterMap.broker[request.Topic] = make([]*ring.Ring, 0, request.TopicPartitionCount) topicList = clusterMap.broker[request.Topic] } if request.TopicPartitionCount >= int32(len(topicList)) { // The partition count has increased. Append enough extra partitions, with offset rings, to our slice for i := int32(len(topicList)); i < request.TopicPartitionCount; i++ { topicList = append(topicList, ring.New(module.intervals)) } } // Advance to the next ring entry (this means the pointer is always at the most recent entry, rather than the // oldest entry) topicList[request.Partition] = topicList[request.Partition].Next() partitionEntry := topicList[request.Partition] if partitionEntry.Value == nil { partitionEntry.Value = &brokerOffset{ Offset: request.Offset, Timestamp: request.Timestamp, } } else { ringval, _ := partitionEntry.Value.(*brokerOffset) ringval.Offset = request.Offset ringval.Timestamp = request.Timestamp } requestLogger.Debug("ok") clusterMap.broker[request.Topic] = topicList } func (module *InMemoryStorage) getBrokerOffset(clusterMap *clusterOffsets, topic string, partition int32, requestLogger *zap.Logger) (int64, int32) { clusterMap.brokerLock.RLock() defer clusterMap.brokerLock.RUnlock() topicPartitionList, ok := clusterMap.broker[topic] if !ok { // We don't know about this topic from the brokers yet - skip consumer offsets for now requestLogger.Debug("dropped", zap.String("reason", "no topic")) return 0, 0 } if partition < 0 { // This should never happen, but if it does, log an warning with the offset information for review requestLogger.Warn("negative partition") return 0, 0 } if partition >= int32(len(topicPartitionList)) { // We know about the topic, but partitions have been expanded and we haven't seen that from the broker yet requestLogger.Debug("dropped", zap.String("reason", "no broker partition")) return 0, 0 } if topicPartitionList[partition].Value == nil { // We know about the topic and partition, but we haven't actually gotten the broker offset yet requestLogger.Debug("dropped", zap.String("reason", "no broker offset")) return 0, 0 } return topicPartitionList[partition].Value.(*brokerOffset).Offset, int32(len(topicPartitionList)) } func (module *InMemoryStorage) getPartitionRing(consumerMap *consumerGroup, topic string, partition int32, partitionCount int32, requestLogger *zap.Logger) *ring.Ring { // Get or create the topic for the consumer consumerTopicMap, ok := consumerMap.topics[topic] if !ok { consumerMap.topics[topic] = make([]*consumerPartition, 0, partitionCount) consumerTopicMap = consumerMap.topics[topic] } // Get the partition specified if int(partition) >= len(consumerTopicMap) { // The partition count must have increased. Append enough extra partitions to our slice for i := int32(len(consumerTopicMap)); i < partitionCount; i++ { consumerTopicMap = append(consumerTopicMap, &consumerPartition{}) } consumerMap.topics[topic] = consumerTopicMap } // Get or create the offsets ring for this partition if consumerTopicMap[partition].offsets == nil { consumerTopicMap[partition].offsets = ring.New(module.intervals) } return consumerTopicMap[partition].offsets } func (module *InMemoryStorage) acceptConsumerGroup(group string) bool { if (module.groupWhitelist != nil) && (!module.groupWhitelist.MatchString(group)) { return false } if (module.groupBlacklist != nil) && module.groupBlacklist.MatchString(group) { return false } return true } func (module *InMemoryStorage) addConsumerOffset(request *protocol.StorageRequest, requestLogger *zap.Logger) { clusterMap, ok := module.offsets[request.Cluster] if !ok { // Ignore offsets for clusters that we don't know about - should never happen anyways requestLogger.Warn("unknown cluster") return } if request.Timestamp < ((time.Now().Unix() - module.expireGroup) * 1000) { requestLogger.Debug("dropped", zap.String("reason", "old offset")) return } if !module.acceptConsumerGroup(request.Group) { requestLogger.Debug("dropped", zap.String("reason", "group not whitelisted")) return } // Get the broker offset for this partition, as well as the partition count brokerOffset, partitionCount := module.getBrokerOffset(&clusterMap, request.Topic, request.Partition, requestLogger) if partitionCount == 0 { // If the returned partitionCount is zero, there was an error that was already logged. Just stop processing return } // Make the consumer group if it does not yet exist clusterMap.consumerLock.Lock() consumerMap, ok := clusterMap.consumer[request.Group] if !ok { clusterMap.consumer[request.Group] = &consumerGroup{ lock: &sync.RWMutex{}, topics: make(map[string][]*consumerPartition), } consumerMap = clusterMap.consumer[request.Group] } clusterMap.consumerLock.Unlock() // For the rest of this, we need the write lock for the consumer group consumerMap.lock.Lock() defer consumerMap.lock.Unlock() // Get the offset ring for this partition - it always points to the earliest offset (or where to insert a new value) consumerPartitionRing := module.getPartitionRing(consumerMap, request.Topic, request.Partition, partitionCount, requestLogger) if consumerPartitionRing.Prev().Value != nil { // If the offset commit is faster than we are allowing (less than the min-distance config), rewind the ring by one spot // This lets us store the offset commit without dropping an old one if (request.Timestamp - consumerPartitionRing.Prev().Value.(*protocol.ConsumerOffset).Timestamp) < (module.minDistance * 1000) { // We have to change both pointers here, as we're essentially rewinding the ring one spot to add this commit consumerPartitionRing = consumerPartitionRing.Prev() consumerMap.topics[request.Topic][request.Partition].offsets = consumerPartitionRing // We also set the timestamp for the request to the STORED timestamp. The reason for this is that if we // update the timestamp to the new timestamp, we may never create a new offset in the ring (consider the // case where someone is auto-committing with a frequency lower than min-distance) request.Timestamp = consumerPartitionRing.Value.(*protocol.ConsumerOffset).Timestamp } } // Calculate the lag against the brokerOffset var partitionLag uint64 if brokerOffset < request.Offset { // Little bit of a hack - because we only get broker offsets periodically, it's possible the consumer offset could be ahead of where we think the broker // is. In this case, just mark it as zero lag. partitionLag = 0 } else { partitionLag = uint64(brokerOffset - request.Offset) } // Update or create the ring value at the current pointer if consumerPartitionRing.Value == nil { consumerPartitionRing.Value = &protocol.ConsumerOffset{ Offset: request.Offset, Timestamp: request.Timestamp, Lag: partitionLag, } } else { ringval, _ := consumerPartitionRing.Value.(*protocol.ConsumerOffset) ringval.Offset = request.Offset ringval.Timestamp = request.Timestamp ringval.Lag = partitionLag } consumerMap.lastCommit = request.Timestamp // Advance the ring pointer requestLogger.Debug("ok", zap.Uint64("lag", partitionLag)) consumerMap.topics[request.Topic][request.Partition].offsets = consumerMap.topics[request.Topic][request.Partition].offsets.Next() } func (module *InMemoryStorage) addConsumerOwner(request *protocol.StorageRequest, requestLogger *zap.Logger) { clusterMap, ok := module.offsets[request.Cluster] if !ok { // Ignore offsets for clusters that we don't know about - should never happen anyways requestLogger.Warn("unknown cluster") return } if !module.acceptConsumerGroup(request.Group) { requestLogger.Debug("dropped", zap.String("reason", "group not whitelisted")) return } // Make the consumer group if it does not yet exist clusterMap.consumerLock.Lock() consumerMap, ok := clusterMap.consumer[request.Group] if !ok { clusterMap.consumer[request.Group] = &consumerGroup{ lock: &sync.RWMutex{}, topics: make(map[string][]*consumerPartition), } consumerMap = clusterMap.consumer[request.Group] } clusterMap.consumerLock.Unlock() // Get the partition count for this partition (we don't need the actual broker offset) _, partitionCount := module.getBrokerOffset(&clusterMap, request.Topic, request.Partition, requestLogger) if partitionCount == 0 { // If the returned partitionCount is zero, there was an error that was already logged. Just stop processing return } // For the rest of this, we need the write lock for the consumer group consumerMap.lock.Lock() defer consumerMap.lock.Unlock() // Get the offset ring for this partition - we don't need it, but it will properly create the topic and partitions for us module.getPartitionRing(consumerMap, request.Topic, request.Partition, partitionCount, requestLogger) if topic, ok := consumerMap.topics[request.Topic]; !ok || (int32(len(topic)) <= request.Partition) { requestLogger.Debug("dropped", zap.String("reason", "no partition")) return } // Write the owner for the given topic/partition requestLogger.Debug("ok") consumerMap.topics[request.Topic][request.Partition].owner = request.Owner consumerMap.topics[request.Topic][request.Partition].clientID = request.ClientID } func (module *InMemoryStorage) clearConsumerOwners(request *protocol.StorageRequest, requestLogger *zap.Logger) { clusterMap, ok := module.offsets[request.Cluster] if !ok { // Ignore metadata for clusters that we don't know about - should never happen anyways requestLogger.Warn("unknown cluster") return } if !module.acceptConsumerGroup(request.Group) { requestLogger.Debug("dropped", zap.String("reason", "group not whitelisted")) return } // Make the consumer group if it does not yet exist clusterMap.consumerLock.Lock() consumerMap, ok := clusterMap.consumer[request.Group] if !ok { // Consumer group doesn't exist, so we can't clear owners for it clusterMap.consumerLock.Unlock() return } clusterMap.consumerLock.Unlock() // For the rest of this, we need the write lock for the consumer group consumerMap.lock.Lock() defer consumerMap.lock.Unlock() for topic, partitions := range consumerMap.topics { for partitionID := range partitions { consumerMap.topics[topic][partitionID].owner = "" consumerMap.topics[topic][partitionID].clientID = "" } } requestLogger.Debug("ok") } func (module *InMemoryStorage) deleteTopic(request *protocol.StorageRequest, requestLogger *zap.Logger) { clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } // Work backwards - remove the topic from consumer groups first for _, consumerMap := range clusterMap.consumer { consumerMap.lock.Lock() // No need to check for existence delete(consumerMap.topics, request.Topic) consumerMap.lock.Unlock() } // Now remove the topic from the broker list clusterMap.brokerLock.Lock() delete(clusterMap.broker, request.Topic) clusterMap.brokerLock.Unlock() requestLogger.Debug("ok") } func (module *InMemoryStorage) deleteGroup(request *protocol.StorageRequest, requestLogger *zap.Logger) { clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } clusterMap.consumerLock.Lock() delete(clusterMap.consumer, request.Group) clusterMap.consumerLock.Unlock() requestLogger.Debug("ok") } func (module *InMemoryStorage) fetchClusterList(request *protocol.StorageRequest, requestLogger *zap.Logger) { defer close(request.Reply) clusterList := make([]string, 0, len(module.offsets)) for cluster := range module.offsets { clusterList = append(clusterList, cluster) } requestLogger.Debug("ok") request.Reply <- clusterList } func (module *InMemoryStorage) fetchTopicList(request *protocol.StorageRequest, requestLogger *zap.Logger) { defer close(request.Reply) clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } clusterMap.brokerLock.RLock() topicList := make([]string, 0, len(clusterMap.broker)) for topic := range clusterMap.broker { topicList = append(topicList, topic) } clusterMap.brokerLock.RUnlock() requestLogger.Debug("ok") request.Reply <- topicList } func (module *InMemoryStorage) fetchConsumerList(request *protocol.StorageRequest, requestLogger *zap.Logger) { defer close(request.Reply) clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } clusterMap.consumerLock.RLock() consumerList := make([]string, 0, len(clusterMap.consumer)) for consumer := range clusterMap.consumer { consumerList = append(consumerList, consumer) } clusterMap.consumerLock.RUnlock() requestLogger.Debug("ok") request.Reply <- consumerList } func (module *InMemoryStorage) fetchTopic(request *protocol.StorageRequest, requestLogger *zap.Logger) { defer close(request.Reply) clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } clusterMap.brokerLock.RLock() topicList, ok := clusterMap.broker[request.Topic] if !ok { requestLogger.Warn("unknown topic") clusterMap.brokerLock.RUnlock() return } offsetList := make([]int64, 0, len(topicList)) for _, partition := range topicList { offsetList = append(offsetList, partition.Value.(*brokerOffset).Offset) } clusterMap.brokerLock.RUnlock() requestLogger.Debug("ok") request.Reply <- offsetList } func getConsumerTopicList(consumerMap *consumerGroup) protocol.ConsumerTopics { topicList := make(protocol.ConsumerTopics) consumerMap.lock.RLock() defer consumerMap.lock.RUnlock() for topic, partitions := range consumerMap.topics { topicList[topic] = make(protocol.ConsumerPartitions, len(partitions)) for partitionID, partition := range partitions { consumerPartition := &protocol.ConsumerPartition{Owner: partition.owner, ClientID: partition.clientID} if partition.offsets != nil { offsetRing := partition.offsets consumerPartition.Offsets = make([]*protocol.ConsumerOffset, offsetRing.Len()) ringPtr := offsetRing for i := 0; i < offsetRing.Len(); i++ { if ringPtr.Value == nil { consumerPartition.Offsets[i] = nil } else { ringval, _ := ringPtr.Value.(*protocol.ConsumerOffset) // Make a copy so that we can release the lock and be safe consumerPartition.Offsets[i] = &protocol.ConsumerOffset{ Offset: ringval.Offset, Lag: ringval.Lag, Timestamp: ringval.Timestamp, } } ringPtr = ringPtr.Next() } } else { consumerPartition.Offsets = make([]*protocol.ConsumerOffset, 0) } topicList[topic][partitionID] = consumerPartition } } return topicList } func (module *InMemoryStorage) fetchConsumer(request *protocol.StorageRequest, requestLogger *zap.Logger) { defer close(request.Reply) clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } clusterMap.consumerLock.RLock() consumerMap, ok := clusterMap.consumer[request.Group] if !ok { requestLogger.Warn("unknown consumer") clusterMap.consumerLock.RUnlock() return } // Lazily purge consumers that haven't committed in longer than the defined interval. Return as a 404 if ((time.Now().Unix() - module.expireGroup) * 1000) > consumerMap.lastCommit { // Swap for a write lock clusterMap.consumerLock.RUnlock() clusterMap.consumerLock.Lock() requestLogger.Debug("purge expired consumer", zap.Int64("last_commit", consumerMap.lastCommit)) delete(clusterMap.consumer, request.Group) clusterMap.consumerLock.Unlock() return } topicList := getConsumerTopicList(consumerMap) clusterMap.consumerLock.RUnlock() // Calculate the current lag for each now. We do this separate from getting the consumer info so we can avoid // locking both the consumers and the brokers at the same time clusterMap.brokerLock.RLock() for topic, partitions := range topicList { topicMap, ok := clusterMap.broker[topic] if !ok { // The topic may have just been deleted, so we'll skip this part and just return the consumer data we have continue } for p, partition := range partitions { // Build the slice of broker offsets to return partition.BrokerOffsets = make([]int64, 0, module.intervals) brokerOffsetPtr := topicMap[p].Next() brokerOffsetPtr.Do(func(item interface{}) { if item != nil { partition.BrokerOffsets = append(partition.BrokerOffsets, item.(*brokerOffset).Offset) } }) if len(partition.Offsets) > 0 { brokerOffset := partition.BrokerOffsets[len(partition.BrokerOffsets)-1] lastOffset := partition.Offsets[len(partition.Offsets)-1] if lastOffset != nil { if brokerOffset < lastOffset.Offset { // Little bit of a hack - because we only get broker offsets periodically, it's possible the consumer offset could be ahead of where we think the broker // is. In this case, just mark it as zero lag. partition.CurrentLag = 0 } else { partition.CurrentLag = uint64(brokerOffset - lastOffset.Offset) } } } } } clusterMap.brokerLock.RUnlock() requestLogger.Debug("ok") request.Reply <- topicList } func (module *InMemoryStorage) fetchConsumersForTopicList(request *protocol.StorageRequest, requestLogger *zap.Logger) { defer close(request.Reply) clusterMap, ok := module.offsets[request.Cluster] if !ok { requestLogger.Warn("unknown cluster") return } clusterMap.consumerLock.RLock() consumerListForTopic := make([]string, 0) for consumerGroup := range clusterMap.consumer { consumerMap := clusterMap.consumer[consumerGroup] topicList := getConsumerTopicList(consumerMap) for topic := range topicList { if topic == request.Topic { consumerListForTopic = append(consumerListForTopic, consumerGroup) break } } } clusterMap.consumerLock.RUnlock() requestLogger.Debug("ok") request.Reply <- consumerListForTopic } burrow-1.2.1/core/internal/storage/inmemory_test.go000066400000000000000000001126001343357346000225000ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package storage import ( "container/ring" "sync" "time" "github.com/stretchr/testify/assert" "testing" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/protocol" ) func fixtureModule(whitelist string, blacklist string) *InMemoryStorage { module := InMemoryStorage{ Log: zap.NewNop(), } module.App = &protocol.ApplicationContext{ StorageChannel: make(chan *protocol.StorageRequest), } viper.Reset() viper.Set("storage.test.class-name", "inmemory") if whitelist != "" { viper.Set("storage.test.group-whitelist", whitelist) } if blacklist != "" { viper.Set("storage.test.group-blacklist", blacklist) } viper.Set("storage.test.min-distance", 1) return &module } func startWithTestCluster(whitelist string) *InMemoryStorage { module := fixtureModule(whitelist, "") // Start needs at least one cluster defined, but it only needs to have a name here viper.Set("cluster.testcluster.class-name", "kafka") viper.Set("cluster.testcluster.servers", []string{"broker1.example.com:1234"}) module.Configure("test", "storage.test") module.Start() return module } func startWithTestBrokerOffsets(whitelist string) *InMemoryStorage { module := startWithTestCluster(whitelist) request := protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "testcluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 1, Offset: 4321, Timestamp: 9876, } module.addBrokerOffset(&request, module.Log) return module } func startWithTestConsumerOffsets(whitelist string, startTime int64) *InMemoryStorage { module := startWithTestBrokerOffsets(whitelist) request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, } for i := 0; i < 10; i++ { request.Offset = int64(1000 + (i * 100)) request.Timestamp = startTime + int64(i*10000) module.addConsumerOffset(&request, module.Log) } return module } func TestInMemoryStorage_ImplementsModule(t *testing.T) { assert.Implements(t, (*protocol.Module)(nil), new(InMemoryStorage)) } func TestInMemoryStorage_ImplementsStorageModule(t *testing.T) { assert.Implements(t, (*Module)(nil), new(InMemoryStorage)) } func TestInMemoryStorage_Configure(t *testing.T) { module := fixtureModule("", "") module.Configure("test", "storage.test") } func TestInMemoryStorage_Configure_DefaultIntervals(t *testing.T) { module := fixtureModule("", "") module.Configure("test", "storage.test") assert.Equal(t, 10, module.intervals, "Default Intervals value of 10 did not get set") } func TestInMemoryStorage_Configure_BadWhitelistRegexp(t *testing.T) { module := fixtureModule("", "") viper.Set("storage.test.group-whitelist", "[") assert.Panics(t, func() { module.Configure("test", "storage.test") }, "The code did not panic") } func TestInMemoryStorage_Configure_BadBlacklistRegexp(t *testing.T) { module := fixtureModule("", "") viper.Set("storage.test.group-blacklist", "[") assert.Panics(t, func() { module.Configure("test", "storage.test") }, "The code did not panic") } func TestInMemoryStorage_Start(t *testing.T) { module := startWithTestCluster("") assert.Len(t, module.offsets, 1, "Module start did not define 1 cluster") } func TestInMemoryStorage_Stop(t *testing.T) { module := startWithTestCluster("") time.Sleep(10 * time.Millisecond) module.Stop() } func TestInMemoryStorage_addBrokerOffset(t *testing.T) { module := startWithTestBrokerOffsets("") topicList, ok := module.offsets["testcluster"].broker["testtopic"] assert.True(t, ok, "Topic not created") assert.Len(t, topicList, 1, "One partition not created") assert.NotNil(t, topicList[0], "brokerOffset ring for p0 not created") assert.NotNil(t, topicList[0].Value, "brokerOffset object for p0 not created") partitonZeroOffset := topicList[0].Value.(*brokerOffset) assert.Equalf(t, int64(4321), partitonZeroOffset.Offset, "Expected offset to be 4321, got %v", partitonZeroOffset.Offset) assert.Equalf(t, int64(9876), partitonZeroOffset.Timestamp, "Expected timestamp to be 9876, got %v", partitonZeroOffset.Timestamp) } func TestInMemoryStorage_addBrokerOffset_ExistingTopic(t *testing.T) { module := startWithTestCluster("") request := protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "testcluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 2, Offset: 4321, Timestamp: 9876, } module.addBrokerOffset(&request, module.Log) request.Partition = 1 request.Offset = 5432 request.Timestamp = 8765 module.addBrokerOffset(&request, module.Log) topicList, ok := module.offsets["testcluster"].broker["testtopic"] assert.True(t, ok, "Topic not created") assert.Len(t, topicList, 2, "Two partitions not created") assert.NotNil(t, topicList[0], "brokerOffset ring for p0 not created") assert.NotNil(t, topicList[0].Value, "brokerOffset object for p0 not created") partitonZeroOffset := topicList[0].Value.(*brokerOffset) assert.Equalf(t, int64(4321), partitonZeroOffset.Offset, "Expected offset for p0 to be 4321, got %v", partitonZeroOffset.Offset) assert.Equalf(t, int64(9876), partitonZeroOffset.Timestamp, "Expected timestamp for p0 to be 9876, got %v", partitonZeroOffset.Timestamp) assert.NotNil(t, topicList[1], "brokerOffset object for p1 not created") partitonOneOffset := topicList[1].Value.(*brokerOffset) assert.Equalf(t, int64(5432), partitonOneOffset.Offset, "Expected offset for p1 to be 5432, got %v", partitonOneOffset.Offset) assert.Equalf(t, int64(8765), partitonOneOffset.Timestamp, "Expected timestamp for p1 to be 8765, got %v", partitonOneOffset.Timestamp) } func TestInMemoryStorage_addBrokerOffset_ExistingPartition(t *testing.T) { module := startWithTestBrokerOffsets("") request := protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "testcluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 1, Offset: 5432, Timestamp: 8765, } module.addBrokerOffset(&request, module.Log) topicList, ok := module.offsets["testcluster"].broker["testtopic"] assert.True(t, ok, "Topic not created") assert.Len(t, topicList, 1, "One partition not created") assert.NotNil(t, topicList[0], "brokerOffset ring for p0 not created") assert.NotNil(t, topicList[0].Value, "brokerOffset object for p0 not created") partitonZeroOffset := topicList[0].Value.(*brokerOffset) assert.Equalf(t, int64(5432), partitonZeroOffset.Offset, "Expected offset for p0 to be 5432, got %v", partitonZeroOffset.Offset) assert.Equalf(t, int64(8765), partitonZeroOffset.Timestamp, "Expected timestamp for p0 to be 8765, got %v", partitonZeroOffset.Timestamp) previousRingItem := topicList[0].Prev() assert.NotNil(t, previousRingItem.Value, "previous brokerOffset object for p0 not created") previousOffset := previousRingItem.Value.(*brokerOffset) assert.Equalf(t, int64(4321), previousOffset.Offset, "Expected offset for p0 to be 4321, got %v", previousOffset.Offset) assert.Equalf(t, int64(9876), previousOffset.Timestamp, "Expected timestamp for p0 to be 9876, got %v", previousOffset.Timestamp) } func TestInMemoryStorage_addBrokerOffset_AddMany(t *testing.T) { module := startWithTestBrokerOffsets("") // Add a lot of offsets request := protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "testcluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 1, Offset: 4321, Timestamp: 9876, } for i := 0; i < 100; i++ { request.Offset = request.Offset + 1 request.Timestamp = request.Timestamp + 1 module.addBrokerOffset(&request, module.Log) } topicList := module.offsets["testcluster"].broker["testtopic"] numOffsets := topicList[0].Len() ringPtr := topicList[0] for i := numOffsets - 1; i >= 0; i-- { ringPtr = ringPtr.Next() assert.NotNilf(t, ringPtr.Value, "Offset %v: brokerOffset object not created", i) val := ringPtr.Value.(*brokerOffset) expectedOffset := request.Offset - int64(i) expectedTimestamp := request.Timestamp - int64(i) assert.Equalf(t, expectedOffset, val.Offset, "Offset %v: Expected offset to be %v, got %v", i, expectedOffset, val.Offset) assert.Equalf(t, expectedTimestamp, val.Timestamp, "Offset %v: Expected timestamp to be %v, got %v", i, expectedTimestamp, val.Timestamp) } } func TestInMemoryStorage_addBrokerOffset_BadCluster(t *testing.T) { module := startWithTestCluster("") request := protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "nocluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 1, Offset: 4321, Timestamp: 9876, } module.addBrokerOffset(&request, module.Log) assert.Len(t, module.offsets, 1, "Extra cluster exists") _, ok := module.offsets["testcluster"].broker["testtopic"] assert.False(t, ok, "Topic created in wrong cluster") } func TestInMemoryStorage_getBrokerOffset(t *testing.T) { module := startWithTestCluster("") request := protocol.StorageRequest{ RequestType: protocol.StorageSetBrokerOffset, Cluster: "testcluster", Topic: "testtopic", Partition: 0, TopicPartitionCount: 2, Offset: 4321, Timestamp: 9876, } module.addBrokerOffset(&request, module.Log) clusterMap := module.offsets["testcluster"] offset, partitions := module.getBrokerOffset(&clusterMap, "testtopic", 0, module.Log) assert.Equalf(t, int64(4321), offset, "Expected offset to be 4321, got %v", offset) assert.Equalf(t, int32(2), partitions, "Expected partitions to be 2, got %v", partitions) offset, partitions = module.getBrokerOffset(&clusterMap, "notopic", 0, module.Log) assert.Equalf(t, int64(0), offset, "Expected offset to be 0, got %v", offset) assert.Equalf(t, int32(0), partitions, "Expected partitions to be 0, got %v", partitions) offset, partitions = module.getBrokerOffset(&clusterMap, "testtopic", 2, module.Log) assert.Equalf(t, int64(0), offset, "Expected offset to be 0, got %v", offset) assert.Equalf(t, int32(0), partitions, "Expected partitions to be 0, got %v", partitions) offset, partitions = module.getBrokerOffset(&clusterMap, "testtopic", 1, module.Log) assert.Equalf(t, int64(0), offset, "Expected offset to be 0, got %v", offset) assert.Equalf(t, int32(0), partitions, "Expected partitions to be 0, got %v", partitions) offset, partitions = module.getBrokerOffset(&clusterMap, "testtopic", -1, module.Log) assert.Equalf(t, int64(0), offset, "Expected offset to be 0, got %v", offset) assert.Equalf(t, int32(0), partitions, "Expected partitions to be 0, got %v", partitions) } func TestInMemoryStorage_addConsumerOffset(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Offset: 2000, Timestamp: startTime + 100000, } module.addConsumerOffset(&request, module.Log) consumerMap, ok := module.offsets["testcluster"].consumer["testgroup"] assert.True(t, ok, "Group not created") partitions, ok := consumerMap.topics["testtopic"] assert.True(t, ok, "Topic not created") assert.Len(t, partitions, 1, "One partition not created") assert.Equal(t, 10, partitions[0].offsets.Len(), "10 offset ring entries not created") assert.Equal(t, "", partitions[0].owner, "Expected owner to be empty") assert.Equal(t, "", partitions[0].clientID, "Expected clientID to be empty") // All the ring values should be not nil r := partitions[0].offsets for i := 0; i < 10; i++ { assert.NotNilf(t, r.Value, "Expected ring value to be NOT nil at position %v", i) assert.IsType(t, new(protocol.ConsumerOffset), r.Value, "Expected ring value to be of type ConsumerOffset") offset := r.Value.(*protocol.ConsumerOffset) offsetValue := int64(1100 + (i * 100)) timestampValue := startTime + 10000 + int64(i*10000) lagValue := uint64(int64(4321) - offsetValue) assert.Equalf(t, offsetValue, offset.Offset, "Expected offset at position %v to be %v, got %v", i, offsetValue, offset.Offset) assert.Equalf(t, timestampValue, offset.Timestamp, "Expected timestamp at position %v to be %v, got %v", i, timestampValue, offset.Timestamp) assert.Equalf(t, lagValue, offset.Lag, "Expected lag at position %v to be %v, got %v", i, lagValue, offset.Lag) r = r.Next() } } func TestInMemoryStorage_addConsumerOffset_Whitelist(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("whitelistedgroup", startTime) // All offsets for the test group should have been dropped _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.False(t, ok, "Group testgroup created when not whitelisted") } func TestInMemoryStorage_addConsumerOffset_TooOld(t *testing.T) { module := startWithTestConsumerOffsets("testgroup", 1000000) // All offsets for the test group should have been dropped as they are too old _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.False(t, ok, "Group testgroup created when offsets are too old") } type testset struct { regexFilter string matchGroups []string noMatchGroups []string } var regexFilterTests = []testset{ {".*", []string{"testgroup", "ok_group", "dash-group", "num02group"}, []string{}}, {"test.*", []string{"testgroup"}, []string{"ok_group", "dash-group", "num02group"}}, {".*[0-9]+.*", []string{"num02group"}, []string{"ok_group", "dash-group", "testgroup"}}, {"onlygroup", []string{"onlygroup"}, []string{"testgroup", "ok_group", "dash-group", "num02group"}}, } func TestInMemoryStorage_acceptConsumerGroup_NoWhitelist(t *testing.T) { for i, testSet := range regexFilterTests { module := fixtureModule(testSet.regexFilter, "") module.Configure("test", "storage.test") for _, group := range testSet.matchGroups { result := module.acceptConsumerGroup(group) assert.Truef(t, result, "TEST %v: Expected group %v to pass", i, group) } for _, group := range testSet.noMatchGroups { result := module.acceptConsumerGroup(group) assert.Falsef(t, result, "TEST %v: Expected group %v to fail", i, group) } } } func TestInMemoryStorage_acceptConsumerGroup_Blacklist(t *testing.T) { // just taking the inverse of TestInMemoryStorage_acceptConsumerGroup_NoWhitelist // so noMatchGroups will return true and matchGroup entries will be false. for i, testSet := range regexFilterTests { module := fixtureModule("", testSet.regexFilter) module.Configure("test", "storage.test") for _, group := range testSet.noMatchGroups { result := module.acceptConsumerGroup(group) assert.Truef(t, result, "TEST %v: Expected group %v to pass", i, group) } for _, group := range testSet.matchGroups { result := module.acceptConsumerGroup(group) assert.Falsef(t, result, "TEST %v: Expected group %v to fail", i, group) } } } func TestInMemoryStorage_addConsumerOffset_MinDistance(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) // This commit violates min-distance and should not cause the ring to advance request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Offset: 2000, Timestamp: startTime + 90001, } module.addConsumerOffset(&request, module.Log) consumerMap, ok := module.offsets["testcluster"].consumer["testgroup"] assert.True(t, ok, "Group not created") partitions, ok := consumerMap.topics["testtopic"] assert.True(t, ok, "Topic not created") assert.Len(t, partitions, 1, "One partition not created") assert.Equal(t, 10, partitions[0].offsets.Len(), "10 offset ring entries not created") // All the ring values should be not nil r := partitions[0].offsets for i := 0; i < 10; i++ { assert.NotNilf(t, r.Value, "Expected ring value to be NOT nil at position %v", i) assert.IsType(t, new(protocol.ConsumerOffset), r.Value, "Expected ring value to be of type ConsumerOffset") offset := r.Value.(*protocol.ConsumerOffset) offsetValue := int64(1000 + (i * 100)) timestampValue := startTime + int64(i*10000) if i == 9 { // The last offset in the ring is the one that got the min-distance update offsetValue = 2000 } lagValue := uint64(int64(4321) - offsetValue) assert.Equalf(t, offsetValue, offset.Offset, "Expected offset at position %v to be %v, got %v", i, offsetValue, offset.Offset) assert.Equalf(t, timestampValue, offset.Timestamp, "Expected timestamp at position %v to be %v, got %v", i, timestampValue, offset.Timestamp) assert.Equalf(t, lagValue, offset.Lag, "Expected lag at position %v to be %v, got %v", i, lagValue, offset.Lag) r = r.Next() } } func TestInMemoryStorage_addConsumerOffset_BadBrokerOffset(t *testing.T) { module := startWithTestBrokerOffsets("") request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "testcluster", Topic: "notopic", Group: "testgroup", Partition: 0, Offset: 3434, Timestamp: 5677, } module.addConsumerOffset(&request, module.Log) // We're only testing one case, as we've previously tested getBrokerOffset to make sure it works completely _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.False(t, ok, "Group created, but offset should have been dropped") } func TestInMemoryStorage_addConsumerOffset_BadCluster(t *testing.T) { module := startWithTestBrokerOffsets("") request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOffset, Cluster: "nocluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Offset: 3434, Timestamp: 5677, } module.addConsumerOffset(&request, module.Log) assert.Len(t, module.offsets, 1, "Extra cluster exists") _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.False(t, ok, "Group created in wrong cluster") } func TestInMemoryStorage_addConsumerOwner(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOwner, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Owner: "testhost.example.com", ClientID: "test_client_id", } module.addConsumerOwner(&request, module.Log) consumerMap, ok := module.offsets["testcluster"].consumer["testgroup"] assert.True(t, ok, "Group not created") partitions, ok := consumerMap.topics["testtopic"] assert.True(t, ok, "Topic not created") assert.Len(t, partitions, 1, "One partition not created") assert.Equal(t, "testhost.example.com", partitions[0].owner, "Expected owner to be testhost.example.com, not %v", partitions[0].owner) assert.Equal(t, "test_client_id", partitions[0].clientID, "Expected clientID to be test_client_id, not %v", partitions[0].clientID) } func TestInMemoryStorage_deleteTopic(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteTopic, Cluster: "testcluster", Topic: "testtopic", } module.deleteTopic(&request, module.Log) _, ok := module.offsets["testcluster"].broker["testtopic"] assert.False(t, ok, "Topic not deleted from broker offsets") consumerMap := module.offsets["testcluster"].consumer["testgroup"] _, ok = consumerMap.topics["testtopic"] assert.False(t, ok, "Topic not deleted from group offsets") } func TestInMemoryStorage_deleteTopic_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteTopic, Cluster: "nocluster", Topic: "testtopic", } module.deleteTopic(&request, module.Log) assert.Len(t, module.offsets, 1, "Extra cluster exists") _, ok := module.offsets["testcluster"].broker["testtopic"] assert.True(t, ok, "Topic deleted in wrong cluster") } func TestInMemoryStorage_deleteTopic_NoTopic(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteTopic, Cluster: "testcluster", Topic: "notopic", } module.deleteTopic(&request, module.Log) _, ok := module.offsets["testcluster"].broker["testtopic"] assert.True(t, ok, "Wrong topic deleted from broker offsets") consumerMap := module.offsets["testcluster"].consumer["testgroup"] _, ok = consumerMap.topics["testtopic"] assert.True(t, ok, "Wrong topic deleted from group offsets") } func TestInMemoryStorage_deleteGroup(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteGroup, Cluster: "testcluster", Group: "testgroup", } module.deleteGroup(&request, module.Log) _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.False(t, ok, "Group not deleted from consumer offsets") } func TestInMemoryStorage_deleteGroup_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteGroup, Cluster: "nocluster", Group: "testgroup", } module.deleteGroup(&request, module.Log) assert.Len(t, module.offsets, 1, "Extra cluster exists") _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.True(t, ok, "Group deleted in wrong cluster") } func TestInMemoryStorage_deleteGroup_NoGroup(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageSetDeleteGroup, Cluster: "testcluster", Group: "nogroup", } module.deleteGroup(&request, module.Log) _, ok := module.offsets["testcluster"].consumer["testgroup"] assert.True(t, ok, "Wrong group deleted from consumer offsets") } func TestInMemoryStorage_fetchClusterList(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchClusters, Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchClusterList(&request, module.Log) response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 1, "One entry not returned") assert.Equalf(t, val[0], "testcluster", "Expected return value to be 'testcluster', not %v", val[0]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchTopicList(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchTopics, Cluster: "testcluster", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchTopicList(&request, module.Log) response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 1, "One entry not returned") assert.Equalf(t, val[0], "testtopic", "Expected return value to be 'testtopic', not %v", val[0]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchTopicList_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchTopics, Cluster: "nocluster", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchTopicList(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumerList(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumers, Cluster: "testcluster", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchConsumerList(&request, module.Log) response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 1, "One entry not returned") assert.Equalf(t, val[0], "testgroup", "Expected return value to be 'testgroup', not %v", val[0]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumerList_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumers, Cluster: "nocluster", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchConsumerList(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchTopic(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchTopic, Cluster: "testcluster", Topic: "testtopic", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchTopic(&request, module.Log) response := <-request.Reply assert.IsType(t, []int64{}, response, "Expected response to be of type []int64") val := response.([]int64) assert.Len(t, val, 1, "One partition not returned") assert.Equalf(t, val[0], int64(4321), "Expected return value to be 4321, not %v", val[0]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchTopic_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchTopic, Cluster: "nocluster", Topic: "testtopic", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchTopic(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchTopic_BadTopic(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchTopic, Cluster: "testcluster", Topic: "notopic", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchTopic(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumer(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) // Set the owner for the test partition request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOwner, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Owner: "testhost.example.com", ClientID: "test_client_id", } module.addConsumerOwner(&request, module.Log) request = protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumer, Cluster: "testcluster", Group: "testgroup", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchConsumer(&request, module.Log) response := <-request.Reply assert.IsType(t, protocol.ConsumerTopics{}, response, "Expected response to be of type map[string][]*protocol.consumerPartition") val := response.(protocol.ConsumerTopics) assert.Len(t, val, 1, "One topic for consumer not returned") _, ok := val["testtopic"] assert.True(t, ok, "Expected response to contain topic testtopic") assert.Len(t, val["testtopic"], 1, "One partition for topic not returned") assert.Equalf(t, uint64(2421), val["testtopic"][0].CurrentLag, "Expected current lag to be 2421, not %v", val["testtopic"][0].CurrentLag) assert.Equalf(t, "testhost.example.com", val["testtopic"][0].Owner, "Expected owner to be testhost.example.com, not %v", val["testtopic"][0].Owner) assert.Equalf(t, "test_client_id", val["testtopic"][0].ClientID, "Expected client_id to be test_client_id, not %v", val["testtopic"][0].ClientID) offsets := val["testtopic"][0].Offsets assert.Lenf(t, offsets, 10, "Expected to get 10 offsets for the partition, not %v", len(offsets)) for i := 0; i < 10; i++ { assert.NotNilf(t, offsets[0], "Expected offset to be NOT nil at position %v", i) offsetValue := int64(1000 + (i * 100)) timestampValue := startTime + int64(i*10000) lagValue := uint64(int64(4321) - offsetValue) assert.Equalf(t, offsetValue, offsets[i].Offset, "Expected offset at position %v to be %v, got %v", i, offsetValue, offsets[i].Offset) assert.Equalf(t, timestampValue, offsets[i].Timestamp, "Expected timestamp at position %v to be %v, got %v", i, timestampValue, offsets[i].Timestamp) assert.Equalf(t, lagValue, offsets[i].Lag, "Expected lag at position %v to be %v, got %v", i, lagValue, offsets[i].Lag) } _, ok = <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumer_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumer, Cluster: "nocluster", Group: "testgroup", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchConsumer(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumer_BadGroup(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumer, Cluster: "testcluster", Group: "nogroup", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchConsumer(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumer_Expired(t *testing.T) { // We can't insert these offsets normally, so we need to mash them into the module module := startWithTestBrokerOffsets("") clusterMap := module.offsets["testcluster"] clusterMap.consumerLock.Lock() clusterMap.consumer["testgroup"] = &consumerGroup{ lock: &sync.RWMutex{}, topics: make(map[string][]*consumerPartition), } consumerMap := clusterMap.consumer["testgroup"] clusterMap.consumerLock.Unlock() consumerMap.lock.Lock() consumerMap.topics["testtopic"] = []*consumerPartition{{offsets: ring.New(module.intervals)}} consumerTopicMap := consumerMap.topics["testtopic"] consumerPartitionRing := consumerTopicMap[0].offsets consumerMap.lock.Unlock() for i := 0; i < 10; i++ { offset := uint64(1000 + (i * 100)) ts := 1000000 + int64(i*10000) consumerPartitionRing.Value = &protocol.ConsumerOffset{ Offset: int64(offset), Timestamp: ts, Lag: 4321 - offset, } consumerMap.lastCommit = ts consumerMap.topics["testtopic"][0].offsets = consumerMap.topics["testtopic"][0].offsets.Next() } request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumer, Cluster: "testcluster", Group: "testgroup", Reply: make(chan interface{}), } // Can't read a reply without concurrency go module.fetchConsumer(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } // TODO: Test for clear consumer offsets, including clear for missing group func TestInMemoryStorage_fetchConsumersForTopic(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) // Set the owners for the test partition request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOwner, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Owner: "testhost.example.com", ClientID: "test_client_id", } module.addConsumerOwner(&request, module.Log) request = protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumersForTopic, Cluster: "testcluster", Topic: "testtopic", Reply: make(chan interface{}), } // Starting request go module.fetchConsumersForTopicList(&request, module.Log) response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 1, "One consumer not returned") assert.Equalf(t, val[0], "testgroup", "Expected return value to be 'testgroup', not %s", val[0]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumersForTopic_MultipleConsumers(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) // Set the owners for the test partition request := protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOwner, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup", Partition: 0, Owner: "testhost.example.com", ClientID: "test_client_id", } module.addConsumerOwner(&request, module.Log) request = protocol.StorageRequest{ RequestType: protocol.StorageSetConsumerOwner, Cluster: "testcluster", Topic: "testtopic", Group: "testgroup2", Partition: 0, Owner: "testhost.example.com", ClientID: "test_client_id", } module.addConsumerOwner(&request, module.Log) request = protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumersForTopic, Cluster: "testcluster", Topic: "testtopic", Reply: make(chan interface{}), } // Starting request go module.fetchConsumersForTopicList(&request, module.Log) response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 2, "Two consumer not returned") assert.True(t, val[0] == "testgroup" || val[0] == "testgroup2", "Expected return value was not given. Found %s", val[0]) assert.True(t, val[1] == "testgroup" || val[1] == "testgroup2", "Expected return value was not given. Found %s", val[1]) _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumersForTopic_NoConsumersForTopic(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumersForTopic, Cluster: "testcluster", Topic: "someNonExistentTopic", Reply: make(chan interface{}), } // Starting request go module.fetchConsumersForTopicList(&request, module.Log) response := <-request.Reply assert.IsType(t, []string{}, response, "Expected response to be of type []string") val := response.([]string) assert.Len(t, val, 0, "Expected no consumers to be returned") _, ok := <-request.Reply assert.False(t, ok, "Expected channel to be closed") } func TestInMemoryStorage_fetchConsumersForTopic_BadCluster(t *testing.T) { startTime := (time.Now().Unix() * 1000) - 100000 module := startWithTestConsumerOffsets("", startTime) request := protocol.StorageRequest{ RequestType: protocol.StorageFetchConsumersForTopic, Cluster: "nonExistentCluster", Topic: "testtopic", Reply: make(chan interface{}), } // Starting request go module.fetchConsumersForTopicList(&request, module.Log) response, ok := <-request.Reply assert.Nil(t, response, "Expected response to be nil") assert.False(t, ok, "Expected channel to be closed") } burrow-1.2.1/core/internal/zookeeper/000077500000000000000000000000001343357346000176125ustar00rootroot00000000000000burrow-1.2.1/core/internal/zookeeper/coordinator.go000066400000000000000000000122571343357346000224730ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package zookeeper - Common Zookeeper subsystem. // The zookeeper subsystem provides a Zookeeper client that is common across all of Burrow, and can be used by other // subsystems to store metadata or coordinate operations between multiple Burrow instances. It is used primarily to // assure that only one Burrow instance is sending notifications at any time. package zookeeper import ( "strings" "sync" "time" "github.com/samuel/go-zookeeper/zk" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) // Coordinator (zookeeper) manages a single Zookeeper connection for other coordinators and modules to make use of in // order to store metadata for Burrow itself. This is not required to connect to the same Zookeeper ensemble as any // specific Kafka cluster. The ZookeeperClient is stored in the application context, as well as the root path that // any modules should create their metadata underneath. // // The coordinator monitors the connection state transitions and signals when the session is expired, and then when it // reconnects. Code that must be aware of session expirations, such as code that makes use of watches, should have a // structure as in the example. type Coordinator struct { App *protocol.ApplicationContext Log *zap.Logger servers []string connectFunc func([]string, time.Duration, *zap.Logger) (protocol.ZookeeperClient, <-chan zk.Event, error) running sync.WaitGroup } // Configure validates that the configuration has a list of servers provided for the Zookeeper ensemble, of the form // host:port. It also checks the provided root path, using a default of "/burrow" if none has been provided. func (zc *Coordinator) Configure() { zc.Log.Info("configuring") if zc.connectFunc == nil { zc.connectFunc = helpers.ZookeeperConnect } // Set and check configs viper.SetDefault("zookeeper.timeout", 6) viper.SetDefault("zookeeper.root-path", "/burrow") zc.servers = viper.GetStringSlice("zookeeper.servers") if len(zc.servers) == 0 { panic("No Zookeeper servers specified") } else if !helpers.ValidateHostList(zc.servers) { panic("Failed to validate Zookeeper servers") } zc.App.ZookeeperRoot = viper.GetString("zookeeper.root-path") if !helpers.ValidateZookeeperPath(zc.App.ZookeeperRoot) { panic("Zookeeper root path is not valid") } zc.running = sync.WaitGroup{} } // Start creates the connection to the Zookeeper ensemble, and assures that the root path exists. Once that is done, // it sets the ZookeeperConnected flag in the application context to true, and creates the ZookeeperExpired condition // flag. It then starts a main loop to watch for connection state changes. func (zc *Coordinator) Start() error { zc.Log.Info("starting") // This ZK client will be shared by other parts of Burrow for things like locks // NOTE - samuel/go-zookeeper does not support chroot, so we pass along the configured root path in config zkConn, connEventChan, err := zc.connectFunc(zc.servers, viper.GetDuration("zookeeper.timeout")*time.Second, zc.Log) if err != nil { zc.Log.Panic("Failure to start zookeeper", zap.String("error", err.Error())) return err } zc.App.Zookeeper = zkConn // Assure that our root path exists err = zc.createRecursive(zc.App.ZookeeperRoot) if err != nil { zc.Log.Error("cannot create root path", zap.Error(err)) return err } zc.App.ZookeeperConnected = true zc.App.ZookeeperExpired = &sync.Cond{L: &sync.Mutex{}} go zc.mainLoop(connEventChan) return nil } // Stop closes the connection to the Zookeeper ensemble and waits for the connection state monitor to exit (which it // will because the event channel will be closed). func (zc *Coordinator) Stop() error { zc.Log.Info("stopping") // This will close the event channel, closing the mainLoop zc.App.Zookeeper.Close() zc.running.Wait() return nil } func (zc *Coordinator) createRecursive(path string) error { if path == "/" { return nil } parts := strings.Split(path, "/") for i := 2; i <= len(parts); i++ { _, err := zc.App.Zookeeper.Create(strings.Join(parts[:i], "/"), []byte{}, 0, zk.WorldACL(zk.PermAll)) // Ignore when the node exists already if (err != nil) && (err != zk.ErrNodeExists) { return err } } return nil } func (zc *Coordinator) mainLoop(eventChan <-chan zk.Event) { zc.running.Add(1) defer zc.running.Done() for event := range eventChan { if event.Type == zk.EventSession { switch event.State { case zk.StateExpired: zc.Log.Error("session expired") zc.App.ZookeeperConnected = false zc.App.ZookeeperExpired.Broadcast() case zk.StateConnected: if !zc.App.ZookeeperConnected { zc.Log.Info("starting session") zc.App.ZookeeperConnected = true } } } } } burrow-1.2.1/core/internal/zookeeper/coordinator_test.go000066400000000000000000000116761343357346000235360ustar00rootroot00000000000000// +build !race /* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package zookeeper import ( "sync" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "testing" "github.com/samuel/go-zookeeper/zk" "github.com/spf13/viper" "go.uber.org/zap" "github.com/linkedin/Burrow/core/internal/helpers" "github.com/linkedin/Burrow/core/protocol" ) func fixtureCoordinator() *Coordinator { coordinator := Coordinator{ Log: zap.NewNop(), } coordinator.App = &protocol.ApplicationContext{ Logger: zap.NewNop(), } viper.Reset() viper.Set("zookeeper.root-path", "/test/path/burrow") viper.Set("zookeeper.servers", []string{"zk.example.com:2181"}) viper.Set("zookeeper.timeout", 5) return &coordinator } func TestCoordinator_ImplementsCoordinator(t *testing.T) { assert.Implements(t, (*protocol.Coordinator)(nil), new(Coordinator)) } func TestCoordinator_Configure(t *testing.T) { coordinator := fixtureCoordinator() coordinator.Configure() assert.NotNil(t, coordinator.connectFunc, "Expected connectFunc to get set") } func TestCoordinator_StartStop(t *testing.T) { coordinator := fixtureCoordinator() // mock the connectFunc to return a mock client mockClient := helpers.MockZookeeperClient{} eventChan := make(chan zk.Event) coordinator.connectFunc = func(servers []string, timeout time.Duration, logger *zap.Logger) (protocol.ZookeeperClient, <-chan zk.Event, error) { return &mockClient, eventChan, nil } mockClient.On("Create", "/test", []byte{}, int32(0), zk.WorldACL(zk.PermAll)).Return("", zk.ErrNodeExists) mockClient.On("Create", "/test/path", []byte{}, int32(0), zk.WorldACL(zk.PermAll)).Return("", zk.ErrNodeExists) mockClient.On("Create", "/test/path/burrow", []byte{}, int32(0), zk.WorldACL(zk.PermAll)).Return("", nil) mockClient.On("Close").Run(func(args mock.Arguments) { close(eventChan) }).Return() coordinator.Configure() err := coordinator.Start() assert.Nil(t, err, "Expected Start to not return an error") assert.Equal(t, &mockClient, coordinator.App.Zookeeper, "Expected App.Zookeeper to be set to the mock client") assert.Equalf(t, "/test/path/burrow", coordinator.App.ZookeeperRoot, "Expected App.ZookeeperRoot to be /test/path/burrow, not %v", coordinator.App.ZookeeperRoot) assert.True(t, coordinator.App.ZookeeperConnected, "Expected App.ZookeeperConnected to be true") assert.NotNil(t, coordinator.App.ZookeeperExpired, "Expected App.ZookeeperExpired to be set") err = coordinator.Stop() assert.Nil(t, err, "Expected Stop to not return an error") } func TestCoordinator_mainLoop(t *testing.T) { coordinator := fixtureCoordinator() coordinator.running = sync.WaitGroup{} coordinator.App.ZookeeperConnected = true coordinator.App.ZookeeperExpired = &sync.Cond{L: &sync.Mutex{}} eventChan := make(chan zk.Event) go coordinator.mainLoop(eventChan) // Nothing should change eventChan <- zk.Event{ Type: zk.EventSession, State: zk.StateDisconnected, } assert.True(t, coordinator.App.ZookeeperConnected, "Expected App.ZookeeperConnected to remain true") // On Expiration, the condition should be set and connected should be false coordinator.App.ZookeeperExpired.L.Lock() eventChan <- zk.Event{ Type: zk.EventSession, State: zk.StateExpired, } coordinator.App.ZookeeperExpired.Wait() coordinator.App.ZookeeperExpired.L.Unlock() assert.False(t, coordinator.App.ZookeeperConnected, "Expected App.ZookeeperConnected to be false") eventChan <- zk.Event{ Type: zk.EventSession, State: zk.StateConnected, } time.Sleep(100 * time.Millisecond) assert.True(t, coordinator.App.ZookeeperConnected, "Expected App.ZookeeperConnected to be true") close(eventChan) coordinator.running.Wait() } // Example for the Coordinator docs on how to do connection state monitoring func ExampleCoordinator_stateMonitoring() { // Ignore me - needed to make the example clean app := &protocol.ApplicationContext{} for { // Wait for the Zookeeper connection to be connected for !app.ZookeeperConnected { // Sleep before looping around to prevent a tight loop time.Sleep(100 * time.Millisecond) continue } // Zookeeper is connected // Do all the work you need to do setting up watches, locks, etc. // Wait on the condition that signals that the session has expired app.ZookeeperExpired.L.Lock() app.ZookeeperExpired.Wait() app.ZookeeperExpired.L.Unlock() // The Zookeeper session has been lost // Do any work that you need to in order to clean up, or stop work that was happening inside a lock // Loop around to wait for the Zookeeper session to be established again } } burrow-1.2.1/core/logger.go000066400000000000000000000134141343357346000156040ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package core import ( "fmt" "io/ioutil" "os" "strconv" "strings" "syscall" "time" "github.com/spf13/viper" "go.uber.org/zap" "go.uber.org/zap/zapcore" "gopkg.in/natefinch/lumberjack.v2" ) // CheckAndCreatePidFile takes a single argument, which is the path to a PID file (a file that contains a single // integer, which is the process ID of a running process). If this file exists, and if the PID is that of a running // process, return false as that indicates another copy of this process is already running. Otherwise, create the // file and write this process's PID to the file and return true. Any error doing this (such as not having permissions // to write the file) will return false. // // This func should be called when Burrow starts to prevent multiple copies from running. func CheckAndCreatePidFile(filename string) bool { // Check if the PID file exists if _, err := os.Stat(filename); !os.IsNotExist(err) { // The file exists, so read it and check if the PID specified is running pidString, err := ioutil.ReadFile(filename) if err != nil { fmt.Printf("Cannot read PID file: %v", err) return false } pid, err := strconv.Atoi(string(pidString)) if err != nil { fmt.Printf("Cannot interpret contents of PID file: %v", err) return false } // Try sending a signal to the process to see if it is still running process, err := os.FindProcess(int(pid)) if err == nil { err = process.Signal(syscall.Signal(0)) if (err == nil) || (err == syscall.EPERM) { // The process exists, so we're going to assume it's an old Burrow and we shouldn't start fmt.Printf("Existing process running on PID %d. Exiting", pid) return false } } } // Create a PID file, replacing any existing one (as we already checked it) pidfile, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { fmt.Printf("Cannot write PID file: %v", err) return false } fmt.Fprintf(pidfile, "%v", os.Getpid()) pidfile.Close() return true } // RemovePidFile takes a single argument, which is the path to a PID file. That file is deleted. This func should be // called when Burrow exits. func RemovePidFile(filename string) { err := os.Remove(filename) if err != nil { fmt.Printf("Failed to remove PID file: %v\n", err) } } // ConfigureLogger returns a configured zap.Logger which can be used by Burrow for all logging. It also returns a // zap.AtomicLevel, which can be used to dynamically adjust the level of the logger. The configuration for the logger // is read from viper, with the following defaults: // // logging.level = info // // If logging.filename (path to the log file) is provided, a rolling log file is set up using lumberjack. The // configuration for that log file is read from viper, with the following defaults: // // logging.maxsize = 100 // logging.maxbackups = 10 // logging.maxage = 30 // logging.use-localtime = false // logging.use-compression = false func ConfigureLogger() (*zap.Logger, *zap.AtomicLevel) { var level zap.AtomicLevel var syncOutput zapcore.WriteSyncer // Set config defaults for logging viper.SetDefault("logging.level", "info") viper.SetDefault("logging.maxsize", 100) viper.SetDefault("logging.maxbackups", 10) viper.SetDefault("logging.maxage", 30) // Create an AtomicLevel that we can use elsewhere to dynamically change the logging level logLevel := viper.GetString("logging.level") switch strings.ToLower(logLevel) { case "", "info": level = zap.NewAtomicLevelAt(zap.InfoLevel) case "debug": level = zap.NewAtomicLevelAt(zap.DebugLevel) case "warn": level = zap.NewAtomicLevelAt(zap.WarnLevel) case "error": level = zap.NewAtomicLevelAt(zap.ErrorLevel) case "panic": level = zap.NewAtomicLevelAt(zap.PanicLevel) case "fatal": level = zap.NewAtomicLevelAt(zap.FatalLevel) default: fmt.Printf("Invalid log level supplied. Defaulting to info: %s", logLevel) level = zap.NewAtomicLevelAt(zap.InfoLevel) } // If a filename has been set, set up a rotating logger. Otherwise, use Stdout logFilename := viper.GetString("logging.filename") if logFilename != "" { syncOutput = zapcore.AddSync(&lumberjack.Logger{ Filename: logFilename, MaxSize: viper.GetInt("logging.maxsize"), MaxBackups: viper.GetInt("logging.maxbackups"), MaxAge: viper.GetInt("logging.maxage"), LocalTime: viper.GetBool("logging.use-localtime"), Compress: viper.GetBool("logging.use-compression"), }) } else { syncOutput = zapcore.Lock(os.Stdout) } core := zapcore.NewCore( zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), syncOutput, level, ) logger := zap.New(core) zap.ReplaceGlobals(logger) return logger, &level } // OpenOutLog takes a single argument, which is the path to a log file. This process's stdout and stderr are redirected // to this log file. The os.File object is returned so that it can be managed. func OpenOutLog(filename string) *os.File { // Move existing out file to a dated file if it exists if _, err := os.Stat(filename); err == nil { if err = os.Rename(filename, filename+"."+time.Now().Format("2006-01-02_15:04:05")); err != nil { fmt.Printf("Cannot move old out file: %v", err) os.Exit(1) } } // Redirect stdout and stderr to out file logFile, _ := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0644) internalDup2(logFile.Fd(), 1) internalDup2(logFile.Fd(), 2) return logFile } burrow-1.2.1/core/open_out_log_linux_arm64.go000066400000000000000000000013421343357346000212430ustar00rootroot00000000000000// +build linux,arm64 /* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package core import ( "syscall" ) // linux_arm64 doesn't have syscall.Dup2, so use // the nearly identical syscall.Dup3 instead func internalDup2(oldfd uintptr, newfd uintptr) error { return syscall.Dup3(int(oldfd), int(newfd), 0) } burrow-1.2.1/core/open_out_log_unix.go000066400000000000000000000012171343357346000200570ustar00rootroot00000000000000// +build !windows // +build !arm64 /* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package core import ( "syscall" ) func internalDup2(oldfd uintptr, newfd uintptr) error { return syscall.Dup2(int(oldfd), int(newfd)) } burrow-1.2.1/core/open_out_log_windows.go000066400000000000000000000015621343357346000205710ustar00rootroot00000000000000// +build windows /* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package core import ( "syscall" ) var ( kernel32 = syscall.MustLoadDLL("kernel32.dll") procSetStdHandle = kernel32.MustFindProc("SetStdHandle") ) func internalDup2(oldfd uintptr, newfd uintptr) error { r0, _, e1 := syscall.Syscall(procSetStdHandle.Addr(), 2, oldfd, newfd, 0) if r0 == 0 { if e1 != 0 { return error(e1) } return syscall.EINVAL } return nil } burrow-1.2.1/core/protocol/000077500000000000000000000000001343357346000156345ustar00rootroot00000000000000burrow-1.2.1/core/protocol/evaluator.go000066400000000000000000000153731343357346000201760ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package protocol import "encoding/json" // EvaluatorRequest is sent over the EvaluatorChannel that is stored in the application context. It is a query for the // status of a group in a cluster. The response to this query is sent over the reply channel. This request is typically // used in the HTTP server and notifier subsystems. type EvaluatorRequest struct { // Reply is the channel over which the evaluator will send the status response. The sender should expect to receive // only one message over this channel for each request, and the channel will not be closed after the response is // sent (to facilitate the notifier, which uses a single channel for all responses) Reply chan *ConsumerGroupStatus // The name of the cluster in which the group is found Cluster string // The name of the group to get the status for Group string // If ShowAll is true, the returned status object contains a partition entry for every partition the group consumes, // regardless of the state of that partition. If false (the default), only partitions that have a status of WARN // or above are returned in the status object. ShowAll bool } // PartitionStatus represents the state of a single consumed partition type PartitionStatus struct { // The topic name for this partition Topic string `json:"topic"` // The partition ID Partition int32 `json:"partition"` // If available (for active new consumers), the consumer host that currently owns this partiton Owner string `json:"owner"` // If available (for active new consumers), the client_id of the consumer that currently owns this partition ClientID string `json:"client_id"` // The status of the partition Status StatusConstant `json:"status"` // A ConsumerOffset object that describes the first (oldest) offset that Burrow is storing for this partition Start *ConsumerOffset `json:"start"` // A ConsumerOffset object that describes the last (latest) offset that Burrow is storing for this partition End *ConsumerOffset `json:"end"` // The current number of messages that the consumer is behind for this partition. This is calculated using the // last committed offset and the current broker end offset CurrentLag uint64 `json:"current_lag"` // A number between 0.0 and 1.0 that describes the percentage complete the offset information is for this partition. // For example, if Burrow has been configured to store 10 offsets, and Burrow has only stored 7 commits for this // partition, Complete will be 0.7 Complete float32 `json:"complete"` } // ConsumerGroupStatus is the response object that is sent in reply to an EvaluatorRequest. It describes the current // status of a single consumer group. type ConsumerGroupStatus struct { // The name of the cluster in which the group exists Cluster string `json:"cluster"` // The name of the consumer group Group string `json:"group"` // The status of the consumer group. This is either NOTFOUND, OK, WARN, or ERR. It is calculated from the highest // Status for the individual partitions Status StatusConstant `json:"status"` // A number between 0.0 and 1.0 that describes the percentage complete the partition information is for this group. // A partition that has a Complete value of less than 1.0 will be treated as zero. Complete float32 `json:"complete"` // A slice of PartitionStatus objects showing individual partition status. If the request ShowAll field was true, // this slice will contain every partition consumed by the group. If ShowAll was false, this slice will only // contain the partitions that have a status of WARN or above. Partitions []*PartitionStatus `json:"partitions"` // A count of the total number of partitions that the group has committed offsets for. Note, this may not be the // same as the total number of partitions consumed by the group, if Burrow has not seen commits for all partitions // yet. TotalPartitions int `json:"partition_count"` // A PartitionStatus object for the partition with the highest CurrentLag value Maxlag *PartitionStatus `json:"maxlag"` // The sum of all partition CurrentLag values for the group TotalLag uint64 `json:"totallag"` } // StatusConstant describes the state of a partition or group as a single value. These values are ordered from least // to most "bad", with zero being reserved to indicate that a group is not found. type StatusConstant int const ( // StatusNotFound indicates that the consumer group does not exist. It is not used for partition status. StatusNotFound StatusConstant = 0 // StatusOK indicates that a partition is in a good state. For a group, it indicates that all partitions are in a // good state. StatusOK StatusConstant = 1 // StatusWarning indicates that a partition is lagging - it is making progress, but falling further behind. For a // group, it indicates that one or more partitions are lagging. StatusWarning StatusConstant = 2 // StatusError indicates that a group has one or more partitions that are in the Stop, Stall, or Rewind states. It // is not used for partition status. StatusError StatusConstant = 3 // StatusStop indicates that the consumer has not committed an offset for that partition in some time, and the lag // is non-zero. It is not used for group status. StatusStop StatusConstant = 4 // StatusStall indicates that the consumer is committing offsets for the partition, but they are not increasing and // the lag is non-zero. It is not used for group status. StatusStall StatusConstant = 5 // StatusRewind indicates that the consumer has committed an offset for the partition that is less than the // previous offset. It is not used for group status. StatusRewind StatusConstant = 6 ) var statusStrings = [...]string{"NOTFOUND", "OK", "WARN", "ERR", "STOP", "STALL", "REWIND"} // String returns a string representation of a StatusConstant func (c StatusConstant) String() string { if (c >= 0) && (c < StatusConstant(len(statusStrings))) { return statusStrings[c] } return "UNKNOWN" } // MarshalText implements the encoding.TextMarshaler interface. The status is the string representation of // StatusConstant func (c StatusConstant) MarshalText() ([]byte, error) { return []byte(c.String()), nil } // MarshalJSON implements the json.Marshaler interface. The status is the string representation of StatusConstant func (c StatusConstant) MarshalJSON() ([]byte, error) { return json.Marshal(c.String()) } burrow-1.2.1/core/protocol/protocol.go000066400000000000000000000237501343357346000200330ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Package protocol - Burrow types and interfaces. // The protocol module provides the definitions for most of the common Burrow types and interfaces that are used in the // rest of the application. The documentation here is primarily targeted at developers of Burrow modules, and not the // end user. package protocol import ( "sync" "github.com/samuel/go-zookeeper/zk" "go.uber.org/zap" ) // ApplicationContext is a structure that holds objects that are used across all coordinators and modules. This is // used in lieu of passing individual arguments to all functions. type ApplicationContext struct { // Logger is a configured zap.Logger instance. It is to be used by the main routine directly, and the main routine // creates loggers for each of the coordinators to use that have fields set to identify that coordinator. // // This field can be set prior to calling core.Start() in order to pre-configure the logger. If it is not set, // core.Start() will set up a default logger using the application config. Logger *zap.Logger // LogLevel is an AtomicLevel instance that has been used to set the default level of the Logger. It is used to // dynamically adjust the logging level (such as via an HTTP call) // // If Logger has been set prior to calling core.Start(), LogLevel must be set as well. LogLevel *zap.AtomicLevel // This is used by the main Burrow routines to signal that the configuration is valid. The rest of the code should // not care about this, as the application will exit if the configuration is not valid. ConfigurationValid bool // This is a ZookeeperClient that can be used by any coordinator or module in order to store metadata about their // operation, or to create locks. Any module that uses this client must honor the ZookeeperRoot, which is the root // path under which all ZNodes should be created. Zookeeper ZookeeperClient ZookeeperRoot string // This is a boolean flag which is set or unset by the Zookeeper coordinator to signal when the connection is // established. It should be used in coordination with the ZookeeperExpired condition to monitor when the session // has expired. This indicates locks have been released or watches must be reset. ZookeeperConnected bool ZookeeperExpired *sync.Cond // This is the channel over which any module should send a consumer group evaluation request. It is serviced by the // evaluator Coordinator, and passed to an appropriate evaluator module. EvaluatorChannel chan *EvaluatorRequest // This is the channel over which any module should send storage requests for storage of offsets and group // information, or to fetch the same information. It is serviced by the storage Coordinator. StorageChannel chan *StorageRequest } // Module is a common interface for all modules so that they can be manipulated by the coordinators in the same way. // The interface provides a way to configure the module, and then methods to start it and stop it safely. Each // coordinator may have its own Module interface definition, as well, that adds specific requirements for that type of // module. // // The struct that implements this interface is expected to have an App and Log literal, at the very least. The App // literal will contain the protocol.ApplicationContext object with resources that the module may use. The Log literal // will be set up with a logger that has fields set that identify the module. These are set up by the module's // coordinator before Configure is called. type Module interface { // Configure is called to initially set up the module. The name of the module, as well as the root string to be // used when looking up configurations with viper, are provided. In this func, the module must completely validate // it's own configuration, and panic if it is not correct. It may also set up data structures that are critical // for the module. It must NOT make any connections to resources outside of the module itself, including either // the storage or evaluator channels in the application context. Configure(name string, configRoot string) // Start is called to start the operation of the module. In this func, the module should make connections to // external resources and start operation. This func must return (any running code must be started as a goroutine). // If there is a problem starting up, the module should stop anything it has already started and return a non-nil // error. Start() error // Stop is called to stop operation of the module. In this func, the module should clean up any goroutines it has // started and close any external connections. While it can return an error if there is a problem, the errors are // mostly ignored. Stop() error } // Coordinator is a common interface for all subsystem coordinators so that the core routine can manage them in a // consistent manner. The interface provides a way to configure the coordinator, and then methods to start it and stop // it safely. It is expected that when any of these funcs are called, the coordinator will then call the corresponding // func on its modules. // // The struct that implements this interface is expected to have an App and Log literal, at the very least. The App // literal will contain the protocol.ApplicationContext object with resources that the coordinator and modules may use. // The Log literal will be set up with a logger that has fields set that identify the coordinator. These are set up by // the core routine before Configure is called. The coordinator can use Log to create the individual loggers for the // modules it controls. type Coordinator interface { // Configure is called to initially set up the coordinator. In this func, it should validate any configurations // that the coordinator requires, and then call the Configure func for each of its modules. If there are any errors // in configuration, it is expected that this call will panic. The coordinator may also set up data structures that // are critical for the subsystem, such as communication channels. It must NOT make any connections to resources // outside of the coordinator itself, including either the storage or evaluator channels in the application context. Configure() // Start is called to start the operation of the coordinator. In this func, the coordinator should call the Start // func for any of its modules, and then start any additional logic the coordinator needs to run. This func must // return (any running code must be started as a goroutine). If there is a problem starting up, the coordinator // should stop anything it has already started and return a non-nil error. Start() error // Stop is called to stop operation of the coordinator. In this func, the coordinator should call the Stop func for // any of its modules, and stop any goroutines that it has started. While it can return an error if there is a // problem, the errors are mostly ignored. Stop() error } // ZookeeperClient is a minimal interface for working with a Zookeeper connection. We provide this interface, rather // than using the underlying library directly, as it makes it easier to test code that uses Zookeeper. This interface // should be expanded with additional methods as needed. // // Note that the interface is specified in the protocol package, rather than in the helper package or the zookeeper // coordinator package, as it has to be referenced by ApplicationContext. Moving it elsewhere generates a dependency // loop. type ZookeeperClient interface { // Close the connection to Zookeeper Close() // For the given path in Zookeeper, return a slice of strings which list the immediate child nodes. This method also // sets a watch on the children of the specified path, providing an event channel that will receive a message when // the watch fires ChildrenW(path string) ([]string, *zk.Stat, <-chan zk.Event, error) // For the given path in Zookeeper, return the data in the node as a byte slice. This method also sets a watch on // the children of the specified path, providing an event channel that will receive a message when the watch fires GetW(path string) ([]byte, *zk.Stat, <-chan zk.Event, error) // For the given path in Zookeeper, return a boolean stating whether or not the node exists. This method also sets // a watch on the node (exists if it does not currently exist, or a data watch otherwise), providing an event // channel that will receive a message when the watch fires ExistsW(path string) (bool, *zk.Stat, <-chan zk.Event, error) // Create makes a new ZNode at the specified path with the contents set to the data byte-slice. Flags can be // provided to specify that this is an ephemeral or sequence node, and an ACL must be provided. If no ACL is\ // desired, specify // zk.WorldACL(zk.PermAll) Create(string, []byte, int32, []zk.ACL) (string, error) // NewLock creates a lock using the provided path. Multiple Zookeeper clients, using the same lock path, can // synchronize with each other to assure that only one client has the lock at any point. NewLock(path string) ZookeeperLock } // ZookeeperLock is an interface for the operation of a lock in Zookeeper. Multiple Zookeeper clients, using the same // lock path, can synchronize with each other to assure that only one client has the lock at any point. type ZookeeperLock interface { // Lock acquires the lock, blocking until it is able to do so, and returns nil. If the lock cannot be acquired, such // as if the session has been lost, a non-nil error will be returned instead. Lock() error // Unlock releases the lock, returning nil. If there is an error releasing the lock, such as if it is not held, an // error is returned instead. Unlock() error } burrow-1.2.1/core/protocol/storage.go000066400000000000000000000175401343357346000176360ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package protocol import "encoding/json" // StorageRequestConstant is used in StorageRequest to indicate the type of request. Numeric ordering is not important type StorageRequestConstant int const ( // StorageSetBrokerOffset is the request type to store a broker offset. Requires Cluster, Topic, Partition, // TopicPartitionCount, and Offset fields StorageSetBrokerOffset StorageRequestConstant = 0 // StorageSetConsumerOffset is the request type to store a consumer offset. Requires Cluster, Group, Topic, // Partition, Offset, and Timestamp fields StorageSetConsumerOffset StorageRequestConstant = 1 // StorageSetConsumerOwner is the request type to store a consumer owner. Requires Cluster, Group, Topic, Partition, // and Owner fields StorageSetConsumerOwner StorageRequestConstant = 2 // StorageSetDeleteTopic is the request type to remove a topic from the broker and all consumers. Requires Cluster, // Group, and Topic fields StorageSetDeleteTopic StorageRequestConstant = 3 // StorageSetDeleteGroup is the request type to remove a consumer group. Requires Cluster and Group fields StorageSetDeleteGroup StorageRequestConstant = 4 // StorageFetchClusters is the request type to retrieve a list of clusters. Requires Reply. Returns a []string StorageFetchClusters StorageRequestConstant = 5 // StorageFetchConsumers is the request type to retrieve a list of consumer groups in a cluster. Requires Reply and // Cluster fields. Returns a []string StorageFetchConsumers StorageRequestConstant = 6 // StorageFetchTopics is the request type to retrieve a list of topics in a cluster. Requires Reply and Cluster // fields. Returns a []string StorageFetchTopics StorageRequestConstant = 7 // StorageFetchConsumer is the request type to retrieve all stored information for a single consumer group. Requires // Reply, Cluster, and Group fields. Returns a ConsumerTopics object StorageFetchConsumer StorageRequestConstant = 8 // StorageFetchTopic is the request type to retrieve the current broker offsets (one per partition) for a topic. // Requires Reply, Cluster, and Topic fields. // Returns a []int64 StorageFetchTopic StorageRequestConstant = 9 // StorageClearConsumerOwners is the request type to remove all partition owner information for a single group. // Requires Cluster and Group fields StorageClearConsumerOwners StorageRequestConstant = 10 // StorageFetchConsumersForTopic is the request type to obtain a list of all consumer groups consuming from a topic. // Returns a []string StorageFetchConsumersForTopic StorageRequestConstant = 11 ) var storageRequestStrings = [...]string{ "StorageSetBrokerOffset", "StorageSetConsumerOffset", "StorageSetConsumerOwner", "StorageSetDeleteTopic", "StorageSetDeleteGroup", "StorageFetchClusters", "StorageFetchConsumers", "StorageFetchTopics", "StorageFetchConsumer", "StorageFetchTopic", "StorageClearConsumerOwners", "StorageFetchConsumersForTopic", } // String returns a string representation of a StorageRequestConstant for logging func (c StorageRequestConstant) String() string { if (c >= 0) && (c < StorageRequestConstant(len(storageRequestStrings))) { return storageRequestStrings[c] } return "UNKNOWN" } // MarshalText implements the encoding.TextMarshaler interface. The status is the string representation of // StorageRequestConstant func (c StorageRequestConstant) MarshalText() ([]byte, error) { return []byte(c.String()), nil } // MarshalJSON implements the json.Marshaler interface. The status is the string representation of // StorageRequestConstant func (c StorageRequestConstant) MarshalJSON() ([]byte, error) { return json.Marshal(c.String()) } // StorageRequest is sent over the StorageChannel that is stored in the application context. It is a query to either // send information to the storage subsystem, or retrieve information from it . The RequestType indiciates the // particular type of request. "Set" and "Clear" requests do not get a response. "Fetch" requests will send a response // over the Reply channel supplied in the request type StorageRequest struct { // The type of request that this struct encapsulates RequestType StorageRequestConstant // If the RequestType is a "Fetch" request, Reply must contain a channel to receive the response on Reply chan interface{} // The name of the cluster to which the request applies. Required for all request types except StorageFetchClusters Cluster string // The name of the consumer group to which the request applies Group string // The name of the topic to which the request applies Topic string // The ID of the partition to which the request applies Partition int32 // For StorageSetBrokerOffset requests, TopicPartitionCount indiciates the total number of partitions for the topic TopicPartitionCount int32 // For StorageSetBrokerOffset and StorageSetConsumerOffset requests, the offset to store Offset int64 // For StorageSetConsumerOffset requests, the timestamp of the offset being stored Timestamp int64 // For StorageSetConsumerOwner requests, a string describing the consumer host that owns the partition Owner string // For StorageSetConsumerOwner requests, a string containing the client_id set by the consumer ClientID string } // ConsumerPartition represents the information stored for a group for a single partition. It is used as part of the // response to a StorageFetchConsumer request type ConsumerPartition struct { // A slice containing a ConsumerOffset object for each offset Burrow has stored for this partition. This can be any // length up to the number of intervals Burrow has been configured to store, depending on how many offset commits // have been seen for this partition Offsets []*ConsumerOffset `json:"offsets"` // A slice containing the history of broker offsets stored for this partition. This is used for evaluation only, // and as such it is not provided when encoding to JSON (for HTTP responses) BrokerOffsets []int64 `json:"-"` // A string that describes the consumer host that currently owns this partition, if the information is available // (for active new consumers) Owner string `json:"owner"` // A string containing the client_id set by the consumer (for active new consumers) ClientID string `json:"client_id"` // The current number of messages that the consumer is behind for this partition. This is calculated using the // last committed offset and the current broker end offset CurrentLag uint64 `json:"current-lag"` } // ConsumerOffset represents a single offset stored. It is used as part of the response to a StorageFetchConsumer // request type ConsumerOffset struct { // The offset that is stored Offset int64 `json:"offset"` // The timestamp at which the offset was committed Timestamp int64 `json:"timestamp"` // The number of messages that the consumer was behind at the time that the offset was committed. This number is // not updated after the offset was committed, so it does not represent the current lag of the consumer. Lag uint64 `json:"lag"` } // ConsumerTopics is the response that is sent for a StorageFetchConsumer request. It is a map of topic names to // ConsumerPartitions objects that describe that topic type ConsumerTopics map[string]ConsumerPartitions // ConsumerPartitions describes all partitions for a single topic. The index indicates the partition ID, and the value // is a pointer to a ConsumerPartition object with the offset information for that partition. type ConsumerPartitions []*ConsumerPartition burrow-1.2.1/docker-compose.yml000066400000000000000000000011261343357346000165000ustar00rootroot00000000000000version: "2" services: burrow: build: . volumes: - ${PWD}/docker-config:/etc/burrow/ - ${PWD}/tmp:/var/tmp/burrow ports: - 8000:8000 depends_on: - zookeeper - kafka restart: always zookeeper: image: wurstmeister/zookeeper ports: - 2181:2181 kafka: image: wurstmeister/kafka ports: - 9092:9092 environment: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/local KAFKA_ADVERTISED_HOST_NAME: kafka KAFKA_ADVERTISED_PORT: 9092 KAFKA_CREATE_TOPICS: "test-topic:2:1,test-topic2:1:1,test-topic3:1:1" burrow-1.2.1/docker-config/000077500000000000000000000000001343357346000155555ustar00rootroot00000000000000burrow-1.2.1/docker-config/burrow.toml000066400000000000000000000011101343357346000177630ustar00rootroot00000000000000[zookeeper] servers=[ "zookeeper:2181" ] timeout=6 root-path="/burrow" [cluster.local] class-name="kafka" servers=[ "kafka:9092" ] topic-refresh=60 offset-refresh=30 [consumer.local] class-name="kafka" cluster="local" servers=[ "kafka:9092" ] group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" group-whitelist="" [consumer.local_zk] class-name="kafka_zk" cluster="local" servers=[ "zookeeper:2181" ] zookeeper-path="/local" zookeeper-timeout=30 group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" group-whitelist="" [httpserver.default] address=":8000" burrow-1.2.1/main.go000066400000000000000000000103461343357346000143220ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ // Burrow provides advanced Kafka Consumer Lag Checking. // It is a monitoring companion for Apache Kafka that provides consumer lag checking as a service without the need for // specifying thresholds. It monitors committed offsets for all consumers and calculates the status of those consumers // on demand. An HTTP endpoint is provided to request status on demand, as well as provide other Kafka cluster // information. There are also configurable notifiers that can send status out via email or HTTP calls to another // service. // // CLI or Library // // Burrow is designed to be run as a standalone application (CLI), and this is what the main package provides. In some // situations it may be better for you to wrap Burrow with another application - for example, in environments where you // have your own application structure to provide configuration and logging. To this end, Burrow can also be used as a // library within another app. // // When embedding Burrow, please refer to https://github.com/linkedin/Burrow/blob/master/main.go for details on what // preparation should happen before starting it. This is the wrapper that provides the CLI interface. The main logic // for Burrow is in the core package, while the protocol package provides some of the common interfaces that are used. // // Additional Documentation // // More documentation on Burrow, including configuration and HTTP requests, can be found at // https://github.com/linkedin/Burrow/wiki package main import ( "flag" "fmt" "os" "os/signal" "runtime" "strings" "syscall" "time" "github.com/spf13/viper" "github.com/linkedin/Burrow/core" ) // exitCode wraps a return value for the application type exitCode struct{ Code int } func handleExit() { if e := recover(); e != nil { if exit, ok := e.(exitCode); ok { if exit.Code != 0 { fmt.Fprintln(os.Stderr, "Burrow failed at", time.Now().Format("January 2, 2006 at 3:04pm (MST)")) } else { fmt.Fprintln(os.Stderr, "Stopped Burrow at", time.Now().Format("January 2, 2006 at 3:04pm (MST)")) } os.Exit(exit.Code) } panic(e) // not an exitCode, bubble up } } func main() { // This makes sure that we panic and run defers correctly defer handleExit() runtime.GOMAXPROCS(runtime.NumCPU()) // The only command line arg is the config file configPath := flag.String("config-dir", ".", "Directory that contains the configuration file") flag.Parse() // Load the configuration from the file viper.SetConfigName("burrow") viper.AddConfigPath(*configPath) fmt.Fprintln(os.Stderr, "Reading configuration from", *configPath) err := viper.ReadInConfig() if err != nil { fmt.Fprintln(os.Stderr, "Failed reading configuration:", err.Error()) panic(exitCode{1}) } // setup viper to be able to read env variables with a configured prefix viper.SetDefault("general.env-var-prefix", "burrow") envPrefix := viper.GetString("general.env-var-prefix") viper.SetEnvPrefix(envPrefix) viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) viper.AutomaticEnv() // Create the PID file to lock out other processes viper.SetDefault("general.pidfile", "burrow.pid") pidFile := viper.GetString("general.pidfile") if !core.CheckAndCreatePidFile(pidFile) { // Any error on checking or creating the PID file causes an immediate exit panic(exitCode{1}) } defer core.RemovePidFile(pidFile) // Set up stderr/stdout to go to a separate log file, if enabled stdoutLogfile := viper.GetString("general.stdout-logfile") if stdoutLogfile != "" { core.OpenOutLog(stdoutLogfile) } // Register signal handlers for exiting exitChannel := make(chan os.Signal, 1) signal.Notify(exitChannel, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) // This triggers handleExit (after other defers), which will then call os.Exit properly panic(exitCode{core.Start(nil, exitChannel)}) } burrow-1.2.1/main_test.go000066400000000000000000000012171343357346000153560ustar00rootroot00000000000000/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ package main import ( "testing" ) // This is just a dummy test function to have a base to start with for Travis func Test_dummy(t *testing.T) { t.Log("Dummy test passed") } burrow-1.2.1/testAndCover.sh000077500000000000000000000017331343357346000160070ustar00rootroot00000000000000#!/bin/bash # This script tests multiple packages and creates a consolidated cover profile # See https://gist.github.com/hailiang/0f22736320abe6be71ce for inspiration. function die() { echo $* exit 1 } export GOPATH=`pwd`:$GOPATH # Initialize profile.cov echo "mode: count" > profile.cov # Initialize error tracking ERROR="" # Get package list PACKAGES=$(find core -type d -not -path '*/\.*') # Test each package and append coverage profile info to profile.cov # Note this is just for coverage. We run the race detector separately because it won't work with count for pkg in $PACKAGES do go test --timeout 5s -covermode=count -coverprofile=profile_tmp.cov github.com/linkedin/Burrow/$pkg || ERROR="Error testing $pkg" if [ -f profile_tmp.cov ] then tail -n +2 profile_tmp.cov >> profile.cov || die "Unable to append coverage for $pkg" rm profile_tmp.cov fi done if [ ! -z "$ERROR" ] then die "Encountered error, last error was: $ERROR" fi