pax_global_header00006660000000000000000000000064137343013340014513gustar00rootroot0000000000000052 comment=f73d9af43600d4c225caf094eddebf0a01c0c2c9 kubernetes-split-yaml-0.3.0/000077500000000000000000000000001373430133400157535ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/.gitattributes000066400000000000000000000000341373430133400206430ustar00rootroot00000000000000vendor/* linguist-generated kubernetes-split-yaml-0.3.0/.github/000077500000000000000000000000001373430133400173135ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/.github/workflows/000077500000000000000000000000001373430133400213505ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/.github/workflows/ci.yaml000066400000000000000000000013551373430133400226330ustar00rootroot00000000000000name: Ci on: push: branches: - 'master' pull_request: branches: - '*' jobs: build: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] runs-on: ${{ matrix.os }} name: Build & Test steps: - name: Checkout uses: actions/checkout@v2 - uses: actions/setup-go@v2-beta with: go-version: '^1.14.0' - name: Download dependencies run: go mod download - name: Build & Test run: | go build -v . go test ./... - name: Run GoReleaser uses: goreleaser/goreleaser-action@master with: args: release --snapshot --skip-publish --rm-dist kubernetes-split-yaml-0.3.0/.github/workflows/coverage.yaml000066400000000000000000000010401373430133400240220ustar00rootroot00000000000000name: Test coverage on: push: jobs: codecov: name: codecov runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - uses: actions/setup-go@v2-beta with: go-version: '^1.14.0' - name: Generate coverage report run: go test ./... -coverprofile=coverage.txt -covermode=atomic - name: Upload coverage report uses: codecov/codecov-action@v1.0.6 with: file: ./coverage.txt flags: unittests kubernetes-split-yaml-0.3.0/.github/workflows/release.yml000066400000000000000000000010751373430133400235160ustar00rootroot00000000000000name: Release with goreleaser on: push: tags: - v*.*.* jobs: build: runs-on: ubuntu-latest name: goreleaser steps: - name: Checkout uses: actions/checkout@v2 - name: Unshallow run: git fetch --prune --unshallow - uses: actions/setup-go@v2-beta with: go-version: '^1.14.0' - name: Release via goreleaser uses: goreleaser/goreleaser-action@master with: args: release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} kubernetes-split-yaml-0.3.0/.gitignore000066400000000000000000000000151373430133400177370ustar00rootroot00000000000000coverage.txt kubernetes-split-yaml-0.3.0/Readme.md000066400000000000000000000035251373430133400174770ustar00rootroot00000000000000# kubernetes-split-yaml [![Build Status](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fmogensen%2Fkubernetes-split-yaml%2Fbadge%3Fref%3Dmaster&style=flat)](https://actions-badge.atrox.dev/mogensen/kubernetes-split-yaml/goto?ref=master) [![Go Report Card](https://goreportcard.com/badge/github.com/mogensen/kubernetes-split-yaml)](https://goreportcard.com/report/github.com/mogensen/kubernetes-split-yaml) [![codecov](https://codecov.io/gh/mogensen/kubernetes-split-yaml/branch/master/graph/badge.svg)](https://codecov.io/gh/mogensen/kubernetes-split-yaml) Split the 'giant yaml file' into one file pr kubernetes resource ## Installation If you have golang installed you can use `go get`. ```bash $ go get -v github.com/mogensen/kubernetes-split-yaml ``` This will download the source and install the binary `kubernetes-split-yaml` ## Usage * Simple invocation ``` $ kubernetes-split-yaml giant-k8s-file.yaml ``` * Modify / filter output filenames ``` # Note by default it'll output 0.2.0 non-hierical files $ kubernetes-split-yaml --help # Get namespaced hierarchy for output files $ kubernetes-split-yaml --template_sel tpl_ns --outdir my-clustername/namespaces giant-k8s-file.yaml # Ditto above, but only for Kubernetes objects starting with "myapp" $ kubernetes-split-yaml --name_re ^myapp --template_sel tpl_ns --outdir my-clustername/namespaces giant-k8s-file.yaml # Ditto above, but only for Deployments and StatefulSets $ kubernetes-split-yaml --kind_re '^(StatefulSet|Deployment)' --name_re ^myapp --template_sel tpl_ns --outdir my-clustername/namespaces giant-k8s-file.yaml ``` ## Contributing Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. Please make sure to update tests as appropriate. ## License [MIT](https://choosealicense.com/licenses/mit/) kubernetes-split-yaml-0.3.0/go.mod000066400000000000000000000005011373430133400170550ustar00rootroot00000000000000module github.com/mogensen/kubernetes-split-yaml go 1.12 require ( github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/google/go-cmp v0.4.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/sirupsen/logrus v1.5.0 github.com/urfave/cli/v2 v2.2.0 gopkg.in/yaml.v2 v2.2.8 ) kubernetes-split-yaml-0.3.0/go.sum000066400000000000000000000063401373430133400171110ustar00rootroot00000000000000github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= kubernetes-split-yaml-0.3.0/kubernetes-resources.go000066400000000000000000000007621373430133400224660ustar00rootroot00000000000000package main import "strings" func getShortName(kind string) string { switch strings.ToLower(kind) { case "service": return "svc" case "serviceaccount": return "sa" case "rolebinding": return "rb" case "clusterrolebinding": return "crb" case "clusterrole": return "cr" case "horizontalpodautoscaler": return "hpa" case "poddisruptionbudget": return "pdb" case "customresourcedefinition": return "crd" case "configmap": return "cm" } return strings.ToLower(kind) } kubernetes-split-yaml-0.3.0/kubernetes-resources_test.go000066400000000000000000000010031373430133400235120ustar00rootroot00000000000000package main import ( "fmt" "testing" ) func Test_getShortName(t *testing.T) { tests := []struct { name string have string want string }{ {have: "ServiceAccount", want: "sa"}, {have: "horizontalpodautoscaler", want: "hpa"}, {have: "Pod", want: "pod"}, } for _, tt := range tests { name := fmt.Sprintf("%s -> %s", tt.have, tt.want) t.Run(name, func(t *testing.T) { if got := getShortName(tt.have); got != tt.want { t.Errorf("getShortName() = %v, want %v", got, tt.want) } }) } } kubernetes-split-yaml-0.3.0/main.go000066400000000000000000000125661373430133400172400ustar00rootroot00000000000000package main import ( "io/ioutil" "path/filepath" "bytes" "fmt" "os" "regexp" "strings" "text/template" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) var log = logrus.New() // Some regexp defaults const FileRe = ".+" // NameRe must be less strict than e.g. NamespaceRe specially because of // - non-namespaced like CRDs (they usually contain dots) // - others like PSPs (containing `:`) const NameRe = "^[^/ ]+$" const KindRe = "^[0-9a-zA-Z]+$" // NamespaceRe: empty or DNSre const NamespaceRe = "^(|[a-z0-9]([-a-z0-9]*[a-z0-9])?)$" // Flags defaults var TemplateFlat = filepath.Join("{{.X.Outdir}}", "{{.Metadata.Name}}-{{.X.ShortKind}}.yaml") var TemplateNS = filepath.Join("{{.X.Outdir}}", "{{.X.NS}}", "{{.Metadata.Name}}.{{.Kind}}.yaml") // cli app flags var appFlags = []cli.Flag{ &cli.StringFlag{ Name: "outdir", Value: "generated", Usage: "output dir", }, &cli.StringFlag{ Name: "template_sel", Value: "tpl_flat", Usage: "pre-set template to use", }, &cli.StringFlag{ Name: "tpl_flat", Value: TemplateFlat, Usage: "tpl_flat gotemplate (.X.Outdir is set to `outdir` flag)", }, &cli.StringFlag{ Name: "tpl_ns", Value: TemplateNS, Usage: "tpl_ns gotemplate (.X.NS equals .Metadata.Namespace or '_no_ns_' if unset)", }, &cli.StringFlag{ Name: "name_re", Value: NameRe, Usage: "Kubernetes API metadata.name to match", }, &cli.StringFlag{ Name: "namespace_re", Value: NamespaceRe, Usage: "Kubernetes API metadata.namespace to match", }, &cli.StringFlag{ Name: "kind_re", Value: KindRe, Usage: "Kubernetes API kind to match", }, &cli.StringFlag{ Name: "file_re", Value: FileRe, Usage: "final output file regex to match", }, } type Filters struct { name string namespace string kind string filename string } func main() { app := cli.NewApp() app.Name = "kubernetes-split-yaml" app.Usage = "Split the 'giant yaml file' into one file pr kubernetes resource" app.Flags = appFlags app.Action = func(c *cli.Context) error { handleFile(c, c.Args().Get(0)) return nil } err := app.Run(os.Args) if err != nil { log.Fatalf("Error running: %v", err) } } func outFile(outdir string, t *template.Template, filters *Filters, m *KubernetesAPI) (string, error) { // Setup m.X. "extended" template-convienent fields ns := m.Metadata.Namespace if m.Metadata.Namespace == "" { ns = "_no_ns_" } m.X.Outdir = outdir m.X.NS = ns m.X.ShortKind = getShortName(m.Kind) buf := new(bytes.Buffer) err := t.Execute(buf, m) if err != nil { return "", fmt.Errorf("Failed to template.Execute on %v, verify template string", m) } filename := buf.String() regexps := []struct { str string match string warn bool }{ // Note: we still keep the KindRe, NameRe, NamespaceRe default REs // for *sanity* verification, to avoid tricks like setting namespace to // "../../etc" {str: m.Kind, match: KindRe, warn: true}, {str: m.Kind, match: filters.kind}, {str: m.Metadata.Name, match: NameRe, warn: true}, {str: m.Metadata.Name, match: filters.name}, {str: m.Metadata.Namespace, match: NamespaceRe, warn: true}, {str: m.Metadata.Namespace, match: filters.namespace}, {str: filename, match: filters.filename}, } for _, re := range regexps { matched, err := regexp.Match(re.match, []byte(re.str)) if err != nil { log.Fatalf("Failed to match regexp '%s'", re.match) } if !matched { if re.warn { log.Warnf("Skipped suspicious object with kind='%s' namespace='%s' name='%s'", m.Kind, m.Metadata.Namespace, m.Metadata.Name) } return "", nil } } return filename, nil } func handleFile(c *cli.Context, file string) { outdir := c.String("outdir") templateSel := c.String("template_sel") outfileTemplate := c.String(templateSel) filters := &Filters{ name: c.String("name_re"), namespace: c.String("namespace_re"), kind: c.String("kind_re"), filename: c.String("file_re"), } tpl, err := template.New("outfile").Parse(outfileTemplate) if err != nil { log.Fatalf("Failed create template from '%s'", outfileTemplate) } files := readAndSplitFile(file) for _, fileContent := range files { m, err := getYamlInfo(fileContent) if err != nil { log.Warnf("Ignoring %v", err) } filename, err := outFile(outdir, tpl, filters, m) if err != nil { log.Fatalf("Failed on outFile: %v", err) } if filename == "" { continue } log.Infof("Creating file: %s", filename) fileDir := filepath.Dir(filename) err = os.MkdirAll(fileDir, os.ModePerm) if err != nil { log.Fatalf("Failed to create directory '%s'", fileDir) } err = ioutil.WriteFile(filename, []byte(fileContent), os.ModePerm) if err != nil { log.Fatalf("Failed creating file %s : %v", filename, err) } } } func readAndSplitFile(file string) []string { var fileContent []byte if file == "-" { c, err := ioutil.ReadAll(os.Stdin) fileContent = c if err != nil { log.Fatalf("Failed reading from stdin: %v", err) } } else { c, err := ioutil.ReadFile(file) fileContent = c if err != nil { log.Fatalf("Failed reading file %s : %v", file, err) } } docs := strings.Split(string(fileContent), "\n---") res := []string{} // Trim whitespace in both ends of each yaml docs. // - Re-add a single newline last for _, doc := range docs { content := strings.TrimSpace(doc) // Ignore empty docs if content != "" { res = append(res, content+LineBreak) } } return res } kubernetes-split-yaml-0.3.0/main_test.go000066400000000000000000000103171373430133400202670ustar00rootroot00000000000000package main import ( "io/ioutil" "os" "path/filepath" "reflect" "testing" "github.com/google/go-cmp/cmp" "github.com/urfave/cli/v2" "text/template" ) func Test_outFile(t *testing.T) { tests := []struct { tname string outdir string template string nameRe string namespaceRe string fileRe string kind string name string namespace string fileExp string }{ { tname: "tpl_flat", outdir: "gen", template: TemplateFlat, nameRe: NameRe, namespaceRe: NamespaceRe, fileRe: FileRe, kind: "Pod", namespace: "foo", name: "bar", fileExp: filepath.Join("gen", "bar-pod.yaml"), }, { tname: "tpl_ns", outdir: "gen", template: TemplateNS, nameRe: NameRe, namespaceRe: NamespaceRe, fileRe: FileRe, kind: "Pod", namespace: "foo", name: "bar", fileExp: filepath.Join("gen", "foo", "bar.Pod.yaml"), }, { tname: "ns/name.kind-otherRe", outdir: "gen", template: TemplateNS, nameRe: NameRe, namespaceRe: NamespaceRe, fileRe: "foo.*bar", kind: "Pod", namespace: "foo", name: "bar", fileExp: filepath.Join("gen", "foo", "bar.Pod.yaml"), }, { tname: "no-ns/name.kind-otherRe", outdir: "gen", template: TemplateNS, nameRe: NameRe, namespaceRe: NamespaceRe, fileRe: FileRe, kind: "ClusterRole", namespace: "", name: "bar", fileExp: filepath.Join("gen", "_no_ns_", "bar.ClusterRole.yaml"), }, { tname: "nonmatching-ns", outdir: "gen", template: TemplateNS, nameRe: NameRe, namespaceRe: "qqq", fileRe: FileRe, kind: "Pod", namespace: "foo", name: "bar", fileExp: "", }, { tname: "no-filename-match", outdir: "gen", template: TemplateNS, nameRe: NameRe, namespaceRe: NamespaceRe, fileRe: "foo.*BAR", kind: "Pod", namespace: "foo", name: "bar", fileExp: "", }, { tname: "no-name-match", outdir: "gen", template: TemplateNS, nameRe: "qqq", namespaceRe: NamespaceRe, fileRe: FileRe, kind: "Pod", namespace: "foo", name: "bar", fileExp: "", }, } for _, tt := range tests { t.Run(tt.tname, func(t *testing.T) { m := &KubernetesAPI{ Kind: tt.kind, } tpl, err := template.New("test").Parse(tt.template) if err != nil { t.Errorf("template: %s", tt.template) } m.Metadata.Name = tt.name m.Metadata.Namespace = tt.namespace filters := &Filters{ name: tt.nameRe, namespace: tt.namespaceRe, filename: tt.fileRe, } got, err := outFile(tt.outdir, tpl, filters, m) if got != tt.fileExp { t.Errorf("outFile() got = '%v', want '%v'", got, tt.fileExp) } }) } } func Test_handleFile(t *testing.T) { // determine input files match, err := filepath.Glob("testdata/*.yaml") if err != nil { t.Fatal(err) } for _, in := range match { out := in + ".golden" runTest(t, in, out) } } func runTest(t *testing.T, in, out string) { t.Run(in, func(t *testing.T) { f := filepath.Base(in) dir, err := ioutil.TempDir(os.TempDir(), f+"-") if err != nil { log.Fatal(err) } defer os.RemoveAll(dir) app := cli.NewApp() app.Flags = appFlags app.Action = func(c *cli.Context) error { handleFile(c, in) wantFiles, err := filepath.Glob(filepath.Join(out, "*")) if err != nil { t.Errorf("could not find test files: %v", err) } gotFiles, err := filepath.Glob(filepath.Join(dir, "*")) if err != nil { t.Errorf("could not find result files: %v", err) } if len(gotFiles) != len(wantFiles) { t.Errorf("handleFile() = %v, want %v", len(gotFiles), len(wantFiles)) } for _, wantFile := range wantFiles { fileName := filepath.Base(wantFile) want, _ := ioutil.ReadFile(wantFile) got, _ := ioutil.ReadFile(filepath.Join(dir, filepath.Base(wantFile))) if !reflect.DeepEqual(got, want) { t.Errorf(fileName + "\n" + cmp.Diff(string(got), string(want))) } } return nil } }) } kubernetes-split-yaml-0.3.0/model.go000066400000000000000000000010311373430133400173750ustar00rootroot00000000000000package main // KubernetesAPI is a minimal struct for unmarshaling kubernetes configs into type KubernetesAPI struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` Metadata struct { Name string `yaml:"name"` Namespace string `yaml:"namespace"` Labels struct { Source string `yaml:"source"` } `yaml:"labels"` } `yaml:"metadata"` // X is not a KubernetesAPI field, but a convienient field for templating purposes X struct { NS string Outdir string ShortKind string } } kubernetes-split-yaml-0.3.0/newline_darwin.go000066400000000000000000000000451373430133400213060ustar00rootroot00000000000000package main const LineBreak = "\n" kubernetes-split-yaml-0.3.0/newline_linux.go000066400000000000000000000000451373430133400211610ustar00rootroot00000000000000package main const LineBreak = "\n" kubernetes-split-yaml-0.3.0/newline_windows.go000066400000000000000000000000471373430133400215160ustar00rootroot00000000000000package main const LineBreak = "\r\n" kubernetes-split-yaml-0.3.0/testdata/000077500000000000000000000000001373430133400175645ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml000066400000000000000000001322171373430133400252430ustar00rootroot00000000000000apiVersion: v1 kind: Namespace metadata: labels: app: collectorforkubernetes name: collectorforkubernetes --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: configurations.collectord.io spec: group: collectord.io versions: - name: v1 served: true storage: true scope: Cluster names: plural: configurations singular: configuration kind: Configuration --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: collectorforkubernetes name: collectorforkubernetes namespace: collectorforkubernetes --- apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: collectorforkubernetes-critical value: 1000000000 --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: collectorforkubernetes name: collectorforkubernetes rules: - apiGroups: ['extensions'] resources: ['podsecuritypolicies'] verbs: ['use'] resourceNames: - privileged - apiGroups: - "" - apps - batch - extensions - monitoring.coreos.com - etcd.database.coreos.com - vault.security.coreos.com - certmanager.k8s.io - kafka.strimzi.io - org.eclipse.che - dynatrace.com - nats.io - syndesis.io - collectord.io resources: - alertmanagers - cronjobs - daemonsets - deployments - endpoints - events - jobs - namespaces - nodes - nodes/metrics - nodes/proxy - pods - prometheuses - replicasets - replicationcontrollers - scheduledjobs - services - statefulsets - vaultservices - etcdclusters - challenges - kafkas - checlusters - oneagents - natsclusters - syndesises - persistentvolumeclaims - configurations verbs: - get - list - watch - nonResourceURLs: - /metrics verbs: - get apiGroups: [] resources: [] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: collectorforkubernetes name: collectorforkubernetes namespace: collectorforkubernetes roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: collectorforkubernetes subjects: - kind: ServiceAccount name: collectorforkubernetes namespace: collectorforkubernetes --- apiVersion: v1 kind: ConfigMap metadata: name: collectorforkubernetes namespace: collectorforkubernetes labels: app: collectorforkubernetes data: 001-general.conf: | # The general configuration is used for all deployments # # Run collector with the flag -conf and specify location of the configuration files. # # You can override all the values using environment variables with the format like # COLLECTOR__=
__= # As an example you can set dataPath in [general] section as # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db # This parameter can be configured using -env-override, set it to empty string to disable this feature [general] # Review EULA https://www.outcoldsolutions.com/docs/license-agreement/ # and accept EULA by changing the value to *true* acceptEULA = false # Location for the database # Collector stores positions of the files and internal state dataPath = ./data/ # log level (accepted values are trace, debug, info, warn, error, fatal) logLevel = info # http server gives access to two endpoints # /healthz # /metrics httpServerBinding = # telemetry report endpoint, set it to empty string to disable telemetry telemetryEndpoint = https://license.outcold.solutions/telemetry/ # license check endpoint licenseEndpoint = https://license.outcold.solutions/license/ # license server through proxy licenseServerProxyUrl = # authentication with basic authorization (user:password) licenseServerProxyBasicAuth = # license key license = # Environment variable $KUBERNETES_NODENAME is used by default to setup hostname # Use value below to override specific name hostname = # Default output for events, logs and metrics # valid values: splunk and devnull # Use devnull by default if you don't want to redirect data defaultOutput = splunk # Default buffer size for file input fileInputBufferSize = 256b # Maximum size of one line the file reader can read fileInputLineMaxSize = 1mb # Include custom fields to attach to every event, in example below every event sent to Splunk will hav # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$ # Better way to configure that is to specify labels for Kubernetes Nodes. # ; fields.my_environment = dev # Identify the cluster if you are planning to monitor multiple clusters fields.kubernetes_cluster = - # Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) # Should be in format ec2Metadata.{desired_field_name} = {url path to read the value} # ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id # ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type # subdomain for the annotations added to the pods, workloads, namespaces or containers, like splunk.collectord.io/.. annotationsSubdomain = # Configure acknowledgement database. # - force fsync on every write to Write-Ahead-Log db.fsync = false # - maximum size of the Write-Ahead-Log db.compactAt = 1M # configure global thruput per second for forwarded logs (metrics are not included) # for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded # from the single Collectord instance to 512Kb per second. # You can configure thruput individually for the logs (including specific for container logs) below thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # connection to kubernetes api [general.kubernetes] # Override service URL for Kubernetes (default is ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}) serviceURL = # Environment variable $KUBERNETES_NODENAME is used by default to setup nodeName # Use it only when you need to override it nodeName = # Configuration to access the API server, # see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod # for details tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # Default timeout for http responses. The streaming/watch requests depend on this timeout. timeout = 30m # (obsolete) In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata # metadataFetchRetry = 5s # (obsolete) In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API # metadataFetchWait = 30s # How long to keep the cache for the recent calls to API server (to limit number of calls when collector discovers new pods) metadataTTL = 30s # regex to find pods podsCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?$ # regex to find containers in the pods containersCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?/(docker-|crio-)?[0-9a-f]{64}(\.scope)?(\/.+)?$ # path to the kubelet root location (use it to discover application logs for emptyDir) # the expected format is `pods/{pod-id}/volumes/kubernetes.io~empty-dir/{volume-name}/_data/` volumesRootDir = /rootfs/var/lib/kubelet/ # You can attach annotations as a metadata, using the format # includeAnnotations.{key} = {regexp} # For example if you want to include all annotations that starts with `prometheus.io` or `example.com` you can include # the following format: # includeAnnotations.1 = ^prometheus\.io.* # includeAnnotations.2 = ^example\.com.* # watch for changes (annotations) in the objects watch.namespaces = v1/namespace watch.deployments = apis/v1/deployment watch.configurations = apis/v1/collectord.io/configuration # watch for pods annotations, setup prometheus collection # for these pods # Addon listens on Pod Network # DaemonSets listen on Host Network [input.prometheus_auto] # disable prometheus auto discovery for pods disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # how often to collect prometheus metrics interval = 60s # include metrics help with the events includeHelp = true # http client timeout timeout = 30s # set output (splunk or devnull, default is [general]defaultOutput) output = # Splunk output [output.splunk] # Splunk HTTP Event Collector url url = # You can specify muiltiple splunk URls with # # urls.0 = https://server1:8088/services/collector/event/1.0 # urls.1 = https://server1:8088/services/collector/event/1.0 # urls.2 = https://server1:8088/services/collector/event/1.0 # # Limitations: # * The urls cannot have different path. # Specify how URL should be picked up (in case if multiple is used) # urlSelection = random|round-robin|random-with-round-robin # where: # * random - choose random url on first selection and after each failure (connection or HTTP status code >= 500) # * round-robin - choose url starting from first one and bump on each failure (connection or HTTP status code >= 500) # * random-with-round-robin - choose random url on first selection and after that in round-robin on each # failure (connection or HTTP status code >= 500) urlSelection = random-with-round-robin # Splunk HTTP Event Collector Token token = # Allow invalid SSL server certificate insecure = false # Path to CA cerificate caPath = # CA Name to verify caName = # path for client certificate (if required) clientCertPath = # path for client key (if required) clientKeyPath = # Events are batched with the maximum size set by batchSize and staying in pipeline for not longer # than set by frequency frequency = 5s batchSize = 768K # limit by the number of events (0 value has no limit on the number of events) events = 50 # Splunk through proxy proxyUrl = # authentication with basic authorization (user:password) proxyBasicAuth = # Splunk acknowledgement url (.../services/collector/ack) ackUrl = # You can specify muiltiple splunk URls for ackUrl # # ackUrls.0 = https://server1:8088/services/collector/ack # ackUrls.1 = https://server1:8088/services/collector/ack # ackUrls.2 = https://server1:8088/services/collector/ack # # Make sure that they in the same order as urls for url, to make sure that this Splunk instance will be # able to acknowledge the payload. # # Limitations: # * The urls cannot have different path. # Enable index acknowledgment ackEnabled = false # Index acknowledgment timeout ackTimeout = 3m # Timeout specifies a time limit for requests made by collector. # The timeout includes connection time, any # redirects, and reading the response body. timeout = 30s # in case when pipeline can post to multiple indexes, we want to avoid posibility of blocking # all pipelines, because just some events have incorrect index dedicatedClientPerIndex = true # in case if some indexes aren't used anymore, how often to destroy the dedicated client # dedicatedClientCleanPeriod = 24h # possible values: RedirectToDefault, Drop, Retry incorrectIndexBehavior = RedirectToDefault # gzip compression level (nocompression, default, 1...9) compressionLevel = default # number of dedicated splunk output threads (to increase throughput above 4k events per second) threads = 1 # if you want to exclude some preindexed fields from events # excludeFields.kubernetes_pod_ip = true 002-daemonset.conf: | # DaemonSet configuration is used for Nodes and Masters. # Connection to the docker host [general.docker] # url for docker API, only unix socket is supported url = unix:///rootfs/var/run/docker.sock # path to docker root folder (can fallback to use folder structure to read docker metadata) dockerRootFolder = /rootfs/var/lib/docker/ # (obsolete) In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata # metadataFetchRetry = 5s # (obsolete) In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API # metadataFetchWait = 30s # (obsolete) In case if collector does not see new events for specific container and with the last metadata refresh # We have not found this container - fow how long we should keep this metadata in cache. # metadataTTL = 5s # Timeout for http responses to docker client. The streaming requests depend on this timeout. timeout = 1m # in case of Kubernetes/OpenShift if you schedule some containers with Docker, but not with the Kubernetes # that allows us to find them (by default finding all containers with name not starting with k8s_) containersNameFilter = ^(([^k])|(k[^8])|(k8[^s])|(k8s[^_])).*$ # regex to find docker container cgroups (helps excluding other cgroups with matched ID) containersCgroupFilter = ^(/([^/\s]+/)*(docker-|docker/)[0-9a-f]{64}(\.scope)?)$ // connection to CRIO [general.cri-o] # url for CRIO API, only unix socket is supported url = unix:///rootfs/var/run/crio/crio.sock # Timeout for http responses to docker client. The streaming requests depend on this timeout. timeout = 1m [general.containerd] runtimePath = /rootfs/var/run/containerd namespace = k8s.io # cgroup input [input.system_stats] # disable system level stats disabled.host = false disabled.cgroup = false # cgroups fs location pathCgroups = /rootfs/sys/fs/cgroup # proc location pathProc = /rootfs/proc # how often to collect cgroup stats statsInterval = 30s # override type type.host = kubernetes_stats_v2_host type.cgroup = kubernetes_stats_v2_cgroup # specify Splunk index index.host = index.cgroup = # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # proc input [input.proc_stats] # disable proc level stats disabled = false # proc location pathProc = /rootfs/proc # how often to collect proc stats statsInterval = 60s # override type type = kubernetes_proc_stats_v2 # specify Splunk index index.host = index.cgroup = # proc filesystem includes by default system threads (there can be over 100 of them) # these stats do not help with the observability # excluding them can reduce the size of the index, performance of the searches and usage of the collector includeSystemThreads = false # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # network stats [input.net_stats] # disable net stats disabled = false # proc path location pathProc = /rootfs/proc # how often to collect net stats statsInterval = 30s # override type type = kubernetes_net_stats_v2 # specify Splunk index index.host = index.cgroup = # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # network socket table [input.net_socket_table] # disable net stats disabled = false # proc path location pathProc = /rootfs/proc # how often to collect net stats statsInterval = 30s # override type type = kubernetes_net_socket_table # specify Splunk index index.host = index.cgroup = # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # group connections by tcp_state, localAddr, remoteAddr (if localPort is not the port it is listening on) # that can significally reduces the amount of events group = true # mount input (collects mount stats where kubelet runtime is stored) [input.mount_stats] # disable system level stats disabled = false # how often to collect mount stats statsInterval = 30s # override type type = kubernetes_mount_stats # specify Splunk index index = # set output (splunk or devnull, default is [general]defaultOutput) output = # Container Log files [input.files] # disable container logs monitoring disabled = false # root location of docker log files # logs are expected in standard docker format like {containerID}/{containerID}-json.log # rotated files path = /rootfs/var/lib/docker/containers/ # root location of CRI-O files # logs are expected in Kubernetes format, like {podID}/{containerName}/0.log crioPath = /rootfs/var/log/pods/ # (obsolete) glob matching pattern for log files # glob = */*-json.log* # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often to look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_logs # specify Splunk index index = # docker splits events when they are larger than 10-100k (depends on the docker version) # we join them together by default and forward to Splunk as one event joinPartialEvents = true # In case if your containers report messages with terminal colors or other escape sequences # you can enable strip for all the containers in one place. # Better is to enable it only for required container with the label collectord.io/strip-terminal-escape-sequences=true stripTerminalEscapeSequences = false # Regexp used for stripping terminal colors, it does not stip all the escape sequences # Read http://man7.org/linux/man-pages/man4/console_codes.4.html for more information stripTerminalEscapeSequencesRegex = (\x1b\[\d{1,3}(;\d{1,3})*m)|(\x07)|(\x1b]\d+(\s\d)?;[^\x07]+\x07)|(.*\x1b\[K) # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for for each container log # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the single container to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Application Logs [input.app_logs] # disable container application logs monitoring disabled = false # root location of mounts (applies to hostPath mounts only), if the hostPath differs inside container from the path on host root = /rootfs/ # how often to review list of available volumes syncInterval = 5s # glob matching pattern for log files glob = *.log* # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often to look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_logs # specify Splunk index index = # we split files using new line character, with this configuration you can specify what defines the new event # after new line eventPatternRegex = ^[^\s] # Maximum interval of messages in pipeline eventPatternMaxInterval = 100ms # Maximum time to wait for the messages in pipeline eventPatternMaxWait = 1s # Maximum message size eventPatternMaxSize = 100kb # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for for each container log # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the single container to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Host logs. Input syslog(.\d+)? files [input.files::syslog] # disable host level logs disabled = false # root location of docker files path = /rootfs/var/log/ # regex matching pattern match = ^(syslog|messages)(.\d+)?$ # limit search only on one level recursive = false # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_host_logs # specify Splunk index index = # field extraction extraction = ^(?P[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P[^\s]+)\s(?P[^:\[]+)(\[(?P\d+)\])?: (.+)$ # timestamp field timestampField = timestamp # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = Jan 2 15:04:05 # Adjust date, if month/day aren't set in format timestampSetMonth = false timestampSetDay = false # timestamp location (if not defined by format) timestampLocation = Local # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded) samplingPercent = -1 # sampling key for hash based sampling (should be regexp with the named match pattern `key`) samplingKey = # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for this files group # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the files in this group to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Host logs. Input all *.log(.\d+)? files [input.files::logs] # disable host level logs disabled = false # root location of log files path = /rootfs/var/log/ # regex matching pattern match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$ # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_host_logs # specify Splunk index index = # field extraction extraction = # timestamp field timestampField = # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = # timestamp location (if not defined by format) timestampLocation = # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded) samplingPercent = -1 # sampling key for hash based sampling (should be regexp with the named match pattern `key`) samplingKey = # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for this files group # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the files in this group to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = [input.journald] # disable host level logs disabled = false # root location of log files path.persistent = /rootfs/var/log/journal/ path.volatile = /rootfs/run/log/journal/ # when reach end of journald, how often to pull pollingInterval = 250ms # if you don't want to forward journald from the beginning, # set the oldest event in relative value, like -14h or -30m or -30s (h/m/s supported) startFromRel = # override type type = kubernetes_host_logs # specify Splunk index index = # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded) samplingPercent = -1 # sampling key (should be regexp with the named match pattern `key`) samplingKey = # how often to reopen the journald to free old files reopenInterval = 1h # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for this files group # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the files in this group to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Pipe to join events (container logs only) [pipe.join] # disable joining event disabled = false # Maximum interval of messages in pipeline maxInterval = 100ms # Maximum time to wait for the messages in pipeline maxWait = 1s # Maximum message size maxSize = 100K # Default pattern to indicate new message (should start not from space) patternRegex = ^[^\s] # Kube API Server has trace messages with multi line events [pipe.join::kube-apiserver] disabled = false matchRegex.kubernetes_container_image = ^gcr.io/google_containers/kube-apiserver-.*$ matchRegex.stream = stderr patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s # (depricated, use annotations for settings up join rules) # Define special event join patterns for matched events # Section consist of [pipe.join::] # [pipe.join::my_app] ## Set match pattern for the fields #; matchRegex.docker_container_image = my_app #; matchRegex.stream = stdout ## All events start from '[' #; patternRegex = ^\[\d+ [input.prometheus::kubelet] # disable prometheus kubelet metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host (environment variables are supported, by default Kubernetes node name is used) host = ${KUBERNETES_NODENAME} # override source source = kubelet # how often to collect prometheus metrics interval = 60s # Prometheus endpoint, multiple values can be specified, collector tries them in order till finding the first # working endpoint. # At first trying to get it through proxy endpoint.1proxy = https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/nodes/${KUBERNETES_NODENAME}/proxy/metrics # In case if cannot get it through proxy, trying localhost endpoint.2http = http://127.0.0.1:10255/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = # [diagnostics::node-entropy] # settings.path = /rootfs/proc/sys/kernel/random/entropy_avail # settings.interval = 1h # settings.threshold = 800 003-daemonset-master.conf: | [input.prometheus::kubernetes-api] # disable prometheus kubernetes-api metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host (environment variables are supported, by default Kubernetes node name is used) host = ${KUBERNETES_NODENAME} # override source source = kubernetes-api # how often to collect prometheus metrics interval = 60s # prometheus endpoint # at first trying to get it from localhost (avoiding load balancer, if multiple api servers) endpoint.1localhost = https://127.0.0.1:6443/metrics # as fallback using proxy endpoint.2kubeapi = https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = # This configuration works if scheduled is bind to the localhost:10251 [input.prometheus::scheduler] # disable prometheus scheduler metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = scheduler # how often to collect prometheus metrics interval = 60s # prometheus endpoint endpoint = http://127.0.0.1:10251/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = # This configuration works if controller-manager is bind to the localhost:10252 [input.prometheus::controller-manager] # disable prometheus controller-manager metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = controller-manager # how often to collect prometheus metrics interval = 60s # prometheus endpoint endpoint = http://127.0.0.1:10252/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = false # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = [input.prometheus::etcd] # disable prometheus etcd metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = etcd # how often to collect prometheus metricd interval = 60s # prometheus endpoint endpoint.http = http://:2379/metrics endpoint.https = https://:2379/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = /rootfs/etc/kubernetes/pki/etcd/ca.crt # client certificate for authentication clientCertPath = /rootfs/etc/kubernetes/pki/apiserver-etcd-client.crt clientKeyPath = /rootfs/etc/kubernetes/pki/apiserver-etcd-client.key # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = 004-addon.conf: | [general] # addons can be run in parallel with agents addon = true [input.kubernetes_events] # disable events disabled = false # override type type = kubernetes_events # specify Splunk index index = # (obsolete, depends on kubernetes timeout) # Set the timeout for how long request to watch events going to hang reading. # eventsWatchTimeout = 30m # (obsolete, depends on kubernetes timeout) # Ignore events last seen later that this duration. # eventsTTL = 12h # set output (splunk or devnull, default is [general]defaultOutput) output = [input.kubernetes_watch::pods] # disable events disabled = false # Set the timeout for how often watch request should refresh the whole list refresh = 10m apiVersion = v1 kind = pod namespace = # override type type = kubernetes_objects # specify Splunk index index = # set output (splunk or devnull, default is [general]defaultOutput) output = --- apiVersion: apps/v1 kind: DaemonSet metadata: name: collectorforkubernetes namespace: collectorforkubernetes labels: app: collectorforkubernetes spec: # Default updateStrategy is OnDelete. For collector RollingUpdate is suitable # When you update configuration updateStrategy: type: RollingUpdate selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes labels: daemon: collectorforkubernetes spec: priorityClassName: collectorforkubernetes-critical dnsPolicy: ClusterFirstWithHostNet hostNetwork: true serviceAccountName: collectorforkubernetes # We run this DaemonSet only for Non-Masters affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/master operator: DoesNotExist tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforkubernetes # Collector version image: docker.io/outcoldsolutions/collectorforkubernetes:5.14.284 imagePullPolicy: Always securityContext: runAsUser: 0 privileged: true # Define your resources if you need. Defaults should be fine for most. # You can lower or increase based on your hosts. resources: limits: cpu: 2 memory: 512Mi requests: cpu: 200m memory: 192Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name volumeMounts: # We store state in /data folder (file positions) - name: collectorforkubernetes-state mountPath: /data # Configuration file deployed with ConfigMap - name: collectorforkubernetes-config mountPath: /config/ readOnly: true # Cgroup filesystem to get metrics - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true # Proc filesystem to get metrics - name: proc mountPath: /rootfs/proc readOnly: true # Location of docker root (for container logs and metadata) - name: docker-root mountPath: /rootfs/var/lib/docker/ readOnly: true mountPropagation: HostToContainer # Docker socket - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true # CRI-O socket (if using CRI-O runtime) - name: crio-unix-socket mountPath: /rootfs/var/run/crio/ readOnly: true # Containerd socket (if using containerd runtime) - name: containerd-unix-socket mountPath: /rootfs/var/run/containerd/ readOnly: true # Host logs location (including CRI-O logs) - name: logs mountPath: /rootfs/var/log/ readOnly: true - name: run-logs mountPath: /rootfs/run/log/ readOnly: true # Application logs - name: volumes-root mountPath: /rootfs/var/lib/kubelet/ readOnly: true mountPropagation: HostToContainer # correct timezone - name: localtime mountPath: /etc/localtime readOnly: true volumes: # We store state directly on host, change this location, if # your persistent volume is somewhere else - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ # Location of docker root (for container logs and metadata) - name: docker-root hostPath: path: /var/lib/docker/ # Location of cgroups file system - name: cgroup hostPath: path: /sys/fs/cgroup # Location of proc file system - name: proc hostPath: path: /proc # Host logs location (including CRI-O logs) - name: logs hostPath: path: /var/log - name: run-logs hostPath: path: /run/log # Docker socket - name: docker-unix-socket hostPath: path: /var/run/docker.sock # CRI-O socket (if using CRI-O runtime) - name: crio-unix-socket hostPath: path: /var/run/crio/ # containerd socket (if using containerd runtime) - name: containerd-unix-socket hostPath: path: /var/run/containerd/ # Location for kubelet mounts, to autodiscover application logs - name: volumes-root hostPath: path: /var/lib/kubelet/ # correct timezone - name: localtime hostPath: path: /etc/localtime # configuration from ConfigMap - name: collectorforkubernetes-config configMap: name: collectorforkubernetes items: - key: 001-general.conf path: 001-general.conf - key: 002-daemonset.conf path: 002-daemonset.conf --- apiVersion: apps/v1 kind: DaemonSet metadata: name: collectorforkubernetes-master namespace: collectorforkubernetes labels: app: collectorforkubernetes spec: updateStrategy: type: RollingUpdate selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes-master labels: daemon: collectorforkubernetes spec: priorityClassName: collectorforkubernetes-critical dnsPolicy: ClusterFirstWithHostNet hostNetwork: true serviceAccountName: collectorforkubernetes affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/master operator: Exists tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforkubernetes image: docker.io/outcoldsolutions/collectorforkubernetes:5.14.284 imagePullPolicy: Always securityContext: runAsUser: 0 privileged: true resources: limits: cpu: 2 memory: 512Mi requests: cpu: 200m memory: 192Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name volumeMounts: - name: collectorforkubernetes-state mountPath: /data - name: collectorforkubernetes-config mountPath: /config/ readOnly: true - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true - name: proc mountPath: /rootfs/proc readOnly: true - name: docker-logs mountPath: /rootfs/var/lib/docker/ readOnly: true mountPropagation: HostToContainer - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true - name: crio-unix-socket mountPath: /rootfs/var/run/crio/ readOnly: true - name: containerd-unix-socket mountPath: /rootfs/var/run/containerd/ readOnly: true - name: logs mountPath: /rootfs/var/log/ readOnly: true - name: run-logs mountPath: /rootfs/run/log/ readOnly: true - name: k8s-certs mountPath: /rootfs/etc/kubernetes/pki/ readOnly: true - name: kubelet-root mountPath: /rootfs/var/lib/kubelet/ readOnly: true mountPropagation: HostToContainer - name: localtime mountPath: /etc/localtime readOnly: true volumes: - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ - name: docker-logs hostPath: path: /var/lib/docker/ - name: cgroup hostPath: path: /sys/fs/cgroup - name: proc hostPath: path: /proc - name: logs hostPath: path: /var/log - name: run-logs hostPath: path: /run/log - name: docker-unix-socket hostPath: path: /var/run/docker.sock - name: crio-unix-socket hostPath: path: /var/run/crio/ - name: containerd-unix-socket hostPath: path: /var/run/containerd/ - name: k8s-certs hostPath: path: /etc/kubernetes/pki/ - name: kubelet-root hostPath: path: /var/lib/kubelet/ - name: localtime hostPath: path: /etc/localtime - name: collectorforkubernetes-config configMap: name: collectorforkubernetes items: - key: 001-general.conf path: 001-general.conf - key: 002-daemonset.conf path: 002-daemonset.conf - key: 003-daemonset-master.conf path: 003-daemonset-master.conf --- apiVersion: apps/v1 kind: Deployment metadata: name: collectorforkubernetes-addon namespace: collectorforkubernetes labels: app: collectorforkubernetes spec: replicas: 1 selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes-addon labels: daemon: collectorforkubernetes spec: priorityClassName: collectorforkubernetes-critical serviceAccountName: collectorforkubernetes containers: - name: collectorforkubernetes image: docker.io/outcoldsolutions/collectorforkubernetes:5.14.284 imagePullPolicy: Always securityContext: runAsUser: 0 privileged: true resources: limits: cpu: 500m memory: 256Mi requests: cpu: 50m memory: 64Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name volumeMounts: - name: collectorforkubernetes-state mountPath: /data - name: collectorforkubernetes-config mountPath: /config/ readOnly: true volumes: - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ - name: collectorforkubernetes-config configMap: name: collectorforkubernetes items: - key: 001-general.conf path: 001-general.conf - key: 004-addon.conf path: 004-addon.confkubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.golden/000077500000000000000000000000001373430133400265615ustar00rootroot00000000000000collectorforkubernetes-addon-deployment.yaml000066400000000000000000000032561373430133400374220ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: apps/v1 kind: Deployment metadata: name: collectorforkubernetes-addon namespace: collectorforkubernetes labels: app: collectorforkubernetes spec: replicas: 1 selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes-addon labels: daemon: collectorforkubernetes spec: priorityClassName: collectorforkubernetes-critical serviceAccountName: collectorforkubernetes containers: - name: collectorforkubernetes image: docker.io/outcoldsolutions/collectorforkubernetes:5.14.284 imagePullPolicy: Always securityContext: runAsUser: 0 privileged: true resources: limits: cpu: 500m memory: 256Mi requests: cpu: 50m memory: 64Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name volumeMounts: - name: collectorforkubernetes-state mountPath: /data - name: collectorforkubernetes-config mountPath: /config/ readOnly: true volumes: - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ - name: collectorforkubernetes-config configMap: name: collectorforkubernetes items: - key: 001-general.conf path: 001-general.conf - key: 004-addon.conf path: 004-addon.conf collectorforkubernetes-cm.yaml000066400000000000000000000776501373430133400345670ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: v1 kind: ConfigMap metadata: name: collectorforkubernetes namespace: collectorforkubernetes labels: app: collectorforkubernetes data: 001-general.conf: | # The general configuration is used for all deployments # # Run collector with the flag -conf and specify location of the configuration files. # # You can override all the values using environment variables with the format like # COLLECTOR__=
__= # As an example you can set dataPath in [general] section as # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db # This parameter can be configured using -env-override, set it to empty string to disable this feature [general] # Review EULA https://www.outcoldsolutions.com/docs/license-agreement/ # and accept EULA by changing the value to *true* acceptEULA = false # Location for the database # Collector stores positions of the files and internal state dataPath = ./data/ # log level (accepted values are trace, debug, info, warn, error, fatal) logLevel = info # http server gives access to two endpoints # /healthz # /metrics httpServerBinding = # telemetry report endpoint, set it to empty string to disable telemetry telemetryEndpoint = https://license.outcold.solutions/telemetry/ # license check endpoint licenseEndpoint = https://license.outcold.solutions/license/ # license server through proxy licenseServerProxyUrl = # authentication with basic authorization (user:password) licenseServerProxyBasicAuth = # license key license = # Environment variable $KUBERNETES_NODENAME is used by default to setup hostname # Use value below to override specific name hostname = # Default output for events, logs and metrics # valid values: splunk and devnull # Use devnull by default if you don't want to redirect data defaultOutput = splunk # Default buffer size for file input fileInputBufferSize = 256b # Maximum size of one line the file reader can read fileInputLineMaxSize = 1mb # Include custom fields to attach to every event, in example below every event sent to Splunk will hav # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$ # Better way to configure that is to specify labels for Kubernetes Nodes. # ; fields.my_environment = dev # Identify the cluster if you are planning to monitor multiple clusters fields.kubernetes_cluster = - # Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) # Should be in format ec2Metadata.{desired_field_name} = {url path to read the value} # ec2Metadata.ec2_instance_id = /latest/meta-data/instance-id # ec2Metadata.ec2_instance_type = /latest/meta-data/instance-type # subdomain for the annotations added to the pods, workloads, namespaces or containers, like splunk.collectord.io/.. annotationsSubdomain = # Configure acknowledgement database. # - force fsync on every write to Write-Ahead-Log db.fsync = false # - maximum size of the Write-Ahead-Log db.compactAt = 1M # configure global thruput per second for forwarded logs (metrics are not included) # for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded # from the single Collectord instance to 512Kb per second. # You can configure thruput individually for the logs (including specific for container logs) below thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # connection to kubernetes api [general.kubernetes] # Override service URL for Kubernetes (default is ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}) serviceURL = # Environment variable $KUBERNETES_NODENAME is used by default to setup nodeName # Use it only when you need to override it nodeName = # Configuration to access the API server, # see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod # for details tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # Default timeout for http responses. The streaming/watch requests depend on this timeout. timeout = 30m # (obsolete) In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata # metadataFetchRetry = 5s # (obsolete) In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API # metadataFetchWait = 30s # How long to keep the cache for the recent calls to API server (to limit number of calls when collector discovers new pods) metadataTTL = 30s # regex to find pods podsCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?$ # regex to find containers in the pods containersCgroupFilter = ^/([^/\s]+/)*kubepods(\.slice)?/((kubepods-)?(burstable|besteffort)(\.slice)?/)?([^/]*)pod([0-9a-f]{32}|[0-9a-f\-_]{36})(\.slice)?/(docker-|crio-)?[0-9a-f]{64}(\.scope)?(\/.+)?$ # path to the kubelet root location (use it to discover application logs for emptyDir) # the expected format is `pods/{pod-id}/volumes/kubernetes.io~empty-dir/{volume-name}/_data/` volumesRootDir = /rootfs/var/lib/kubelet/ # You can attach annotations as a metadata, using the format # includeAnnotations.{key} = {regexp} # For example if you want to include all annotations that starts with `prometheus.io` or `example.com` you can include # the following format: # includeAnnotations.1 = ^prometheus\.io.* # includeAnnotations.2 = ^example\.com.* # watch for changes (annotations) in the objects watch.namespaces = v1/namespace watch.deployments = apis/v1/deployment watch.configurations = apis/v1/collectord.io/configuration # watch for pods annotations, setup prometheus collection # for these pods # Addon listens on Pod Network # DaemonSets listen on Host Network [input.prometheus_auto] # disable prometheus auto discovery for pods disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # how often to collect prometheus metrics interval = 60s # include metrics help with the events includeHelp = true # http client timeout timeout = 30s # set output (splunk or devnull, default is [general]defaultOutput) output = # Splunk output [output.splunk] # Splunk HTTP Event Collector url url = # You can specify muiltiple splunk URls with # # urls.0 = https://server1:8088/services/collector/event/1.0 # urls.1 = https://server1:8088/services/collector/event/1.0 # urls.2 = https://server1:8088/services/collector/event/1.0 # # Limitations: # * The urls cannot have different path. # Specify how URL should be picked up (in case if multiple is used) # urlSelection = random|round-robin|random-with-round-robin # where: # * random - choose random url on first selection and after each failure (connection or HTTP status code >= 500) # * round-robin - choose url starting from first one and bump on each failure (connection or HTTP status code >= 500) # * random-with-round-robin - choose random url on first selection and after that in round-robin on each # failure (connection or HTTP status code >= 500) urlSelection = random-with-round-robin # Splunk HTTP Event Collector Token token = # Allow invalid SSL server certificate insecure = false # Path to CA cerificate caPath = # CA Name to verify caName = # path for client certificate (if required) clientCertPath = # path for client key (if required) clientKeyPath = # Events are batched with the maximum size set by batchSize and staying in pipeline for not longer # than set by frequency frequency = 5s batchSize = 768K # limit by the number of events (0 value has no limit on the number of events) events = 50 # Splunk through proxy proxyUrl = # authentication with basic authorization (user:password) proxyBasicAuth = # Splunk acknowledgement url (.../services/collector/ack) ackUrl = # You can specify muiltiple splunk URls for ackUrl # # ackUrls.0 = https://server1:8088/services/collector/ack # ackUrls.1 = https://server1:8088/services/collector/ack # ackUrls.2 = https://server1:8088/services/collector/ack # # Make sure that they in the same order as urls for url, to make sure that this Splunk instance will be # able to acknowledge the payload. # # Limitations: # * The urls cannot have different path. # Enable index acknowledgment ackEnabled = false # Index acknowledgment timeout ackTimeout = 3m # Timeout specifies a time limit for requests made by collector. # The timeout includes connection time, any # redirects, and reading the response body. timeout = 30s # in case when pipeline can post to multiple indexes, we want to avoid posibility of blocking # all pipelines, because just some events have incorrect index dedicatedClientPerIndex = true # in case if some indexes aren't used anymore, how often to destroy the dedicated client # dedicatedClientCleanPeriod = 24h # possible values: RedirectToDefault, Drop, Retry incorrectIndexBehavior = RedirectToDefault # gzip compression level (nocompression, default, 1...9) compressionLevel = default # number of dedicated splunk output threads (to increase throughput above 4k events per second) threads = 1 # if you want to exclude some preindexed fields from events # excludeFields.kubernetes_pod_ip = true 002-daemonset.conf: | # DaemonSet configuration is used for Nodes and Masters. # Connection to the docker host [general.docker] # url for docker API, only unix socket is supported url = unix:///rootfs/var/run/docker.sock # path to docker root folder (can fallback to use folder structure to read docker metadata) dockerRootFolder = /rootfs/var/lib/docker/ # (obsolete) In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata # metadataFetchRetry = 5s # (obsolete) In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API # metadataFetchWait = 30s # (obsolete) In case if collector does not see new events for specific container and with the last metadata refresh # We have not found this container - fow how long we should keep this metadata in cache. # metadataTTL = 5s # Timeout for http responses to docker client. The streaming requests depend on this timeout. timeout = 1m # in case of Kubernetes/OpenShift if you schedule some containers with Docker, but not with the Kubernetes # that allows us to find them (by default finding all containers with name not starting with k8s_) containersNameFilter = ^(([^k])|(k[^8])|(k8[^s])|(k8s[^_])).*$ # regex to find docker container cgroups (helps excluding other cgroups with matched ID) containersCgroupFilter = ^(/([^/\s]+/)*(docker-|docker/)[0-9a-f]{64}(\.scope)?)$ // connection to CRIO [general.cri-o] # url for CRIO API, only unix socket is supported url = unix:///rootfs/var/run/crio/crio.sock # Timeout for http responses to docker client. The streaming requests depend on this timeout. timeout = 1m [general.containerd] runtimePath = /rootfs/var/run/containerd namespace = k8s.io # cgroup input [input.system_stats] # disable system level stats disabled.host = false disabled.cgroup = false # cgroups fs location pathCgroups = /rootfs/sys/fs/cgroup # proc location pathProc = /rootfs/proc # how often to collect cgroup stats statsInterval = 30s # override type type.host = kubernetes_stats_v2_host type.cgroup = kubernetes_stats_v2_cgroup # specify Splunk index index.host = index.cgroup = # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # proc input [input.proc_stats] # disable proc level stats disabled = false # proc location pathProc = /rootfs/proc # how often to collect proc stats statsInterval = 60s # override type type = kubernetes_proc_stats_v2 # specify Splunk index index.host = index.cgroup = # proc filesystem includes by default system threads (there can be over 100 of them) # these stats do not help with the observability # excluding them can reduce the size of the index, performance of the searches and usage of the collector includeSystemThreads = false # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # network stats [input.net_stats] # disable net stats disabled = false # proc path location pathProc = /rootfs/proc # how often to collect net stats statsInterval = 30s # override type type = kubernetes_net_stats_v2 # specify Splunk index index.host = index.cgroup = # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # network socket table [input.net_socket_table] # disable net stats disabled = false # proc path location pathProc = /rootfs/proc # how often to collect net stats statsInterval = 30s # override type type = kubernetes_net_socket_table # specify Splunk index index.host = index.cgroup = # set output (splunk or devnull, default is [general]defaultOutput) output.host = output.cgroup = # group connections by tcp_state, localAddr, remoteAddr (if localPort is not the port it is listening on) # that can significally reduces the amount of events group = true # mount input (collects mount stats where kubelet runtime is stored) [input.mount_stats] # disable system level stats disabled = false # how often to collect mount stats statsInterval = 30s # override type type = kubernetes_mount_stats # specify Splunk index index = # set output (splunk or devnull, default is [general]defaultOutput) output = # Container Log files [input.files] # disable container logs monitoring disabled = false # root location of docker log files # logs are expected in standard docker format like {containerID}/{containerID}-json.log # rotated files path = /rootfs/var/lib/docker/containers/ # root location of CRI-O files # logs are expected in Kubernetes format, like {podID}/{containerName}/0.log crioPath = /rootfs/var/log/pods/ # (obsolete) glob matching pattern for log files # glob = */*-json.log* # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often to look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_logs # specify Splunk index index = # docker splits events when they are larger than 10-100k (depends on the docker version) # we join them together by default and forward to Splunk as one event joinPartialEvents = true # In case if your containers report messages with terminal colors or other escape sequences # you can enable strip for all the containers in one place. # Better is to enable it only for required container with the label collectord.io/strip-terminal-escape-sequences=true stripTerminalEscapeSequences = false # Regexp used for stripping terminal colors, it does not stip all the escape sequences # Read http://man7.org/linux/man-pages/man4/console_codes.4.html for more information stripTerminalEscapeSequencesRegex = (\x1b\[\d{1,3}(;\d{1,3})*m)|(\x07)|(\x1b]\d+(\s\d)?;[^\x07]+\x07)|(.*\x1b\[K) # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for for each container log # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the single container to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Application Logs [input.app_logs] # disable container application logs monitoring disabled = false # root location of mounts (applies to hostPath mounts only), if the hostPath differs inside container from the path on host root = /rootfs/ # how often to review list of available volumes syncInterval = 5s # glob matching pattern for log files glob = *.log* # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often to look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_logs # specify Splunk index index = # we split files using new line character, with this configuration you can specify what defines the new event # after new line eventPatternRegex = ^[^\s] # Maximum interval of messages in pipeline eventPatternMaxInterval = 100ms # Maximum time to wait for the messages in pipeline eventPatternMaxWait = 1s # Maximum message size eventPatternMaxSize = 100kb # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for for each container log # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the single container to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Host logs. Input syslog(.\d+)? files [input.files::syslog] # disable host level logs disabled = false # root location of docker files path = /rootfs/var/log/ # regex matching pattern match = ^(syslog|messages)(.\d+)?$ # limit search only on one level recursive = false # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_host_logs # specify Splunk index index = # field extraction extraction = ^(?P[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P[^\s]+)\s(?P[^:\[]+)(\[(?P\d+)\])?: (.+)$ # timestamp field timestampField = timestamp # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = Jan 2 15:04:05 # Adjust date, if month/day aren't set in format timestampSetMonth = false timestampSetDay = false # timestamp location (if not defined by format) timestampLocation = Local # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded) samplingPercent = -1 # sampling key for hash based sampling (should be regexp with the named match pattern `key`) samplingKey = # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for this files group # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the files in this group to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Host logs. Input all *.log(.\d+)? files [input.files::logs] # disable host level logs disabled = false # root location of log files path = /rootfs/var/log/ # regex matching pattern match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$ # files are read using polling schema, when reach the EOF how often to check if files got updated pollingInterval = 250ms # how often o look for the new files under logs path walkingInterval = 5s # include verbose fields in events (file offset) verboseFields = false # override type type = kubernetes_host_logs # specify Splunk index index = # field extraction extraction = # timestamp field timestampField = # format for timestamp # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006` timestampFormat = # timestamp location (if not defined by format) timestampLocation = # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded) samplingPercent = -1 # sampling key for hash based sampling (should be regexp with the named match pattern `key`) samplingKey = # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for this files group # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the files in this group to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = [input.journald] # disable host level logs disabled = false # root location of log files path.persistent = /rootfs/var/log/journal/ path.volatile = /rootfs/run/log/journal/ # when reach end of journald, how often to pull pollingInterval = 250ms # if you don't want to forward journald from the beginning, # set the oldest event in relative value, like -14h or -30m or -30s (h/m/s supported) startFromRel = # override type type = kubernetes_host_logs # specify Splunk index index = # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded) samplingPercent = -1 # sampling key (should be regexp with the named match pattern `key`) samplingKey = # how often to reopen the journald to free old files reopenInterval = 1h # set output (splunk or devnull, default is [general]defaultOutput) output = # configure default thruput per second for this files group # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded # from the files in this group to 128Kb per second. thruputPerSecond = # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events # older than 7 days tooOldEvents = # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future tooNewEvents = # Pipe to join events (container logs only) [pipe.join] # disable joining event disabled = false # Maximum interval of messages in pipeline maxInterval = 100ms # Maximum time to wait for the messages in pipeline maxWait = 1s # Maximum message size maxSize = 100K # Default pattern to indicate new message (should start not from space) patternRegex = ^[^\s] # Kube API Server has trace messages with multi line events [pipe.join::kube-apiserver] disabled = false matchRegex.kubernetes_container_image = ^gcr.io/google_containers/kube-apiserver-.*$ matchRegex.stream = stderr patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s # (depricated, use annotations for settings up join rules) # Define special event join patterns for matched events # Section consist of [pipe.join::] # [pipe.join::my_app] ## Set match pattern for the fields #; matchRegex.docker_container_image = my_app #; matchRegex.stream = stdout ## All events start from '[' #; patternRegex = ^\[\d+ [input.prometheus::kubelet] # disable prometheus kubelet metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host (environment variables are supported, by default Kubernetes node name is used) host = ${KUBERNETES_NODENAME} # override source source = kubelet # how often to collect prometheus metrics interval = 60s # Prometheus endpoint, multiple values can be specified, collector tries them in order till finding the first # working endpoint. # At first trying to get it through proxy endpoint.1proxy = https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/nodes/${KUBERNETES_NODENAME}/proxy/metrics # In case if cannot get it through proxy, trying localhost endpoint.2http = http://127.0.0.1:10255/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = # [diagnostics::node-entropy] # settings.path = /rootfs/proc/sys/kernel/random/entropy_avail # settings.interval = 1h # settings.threshold = 800 003-daemonset-master.conf: | [input.prometheus::kubernetes-api] # disable prometheus kubernetes-api metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host (environment variables are supported, by default Kubernetes node name is used) host = ${KUBERNETES_NODENAME} # override source source = kubernetes-api # how often to collect prometheus metrics interval = 60s # prometheus endpoint # at first trying to get it from localhost (avoiding load balancer, if multiple api servers) endpoint.1localhost = https://127.0.0.1:6443/metrics # as fallback using proxy endpoint.2kubeapi = https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token # server certificate for certificate validation certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = # This configuration works if scheduled is bind to the localhost:10251 [input.prometheus::scheduler] # disable prometheus scheduler metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = scheduler # how often to collect prometheus metrics interval = 60s # prometheus endpoint endpoint = http://127.0.0.1:10251/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = # This configuration works if controller-manager is bind to the localhost:10252 [input.prometheus::controller-manager] # disable prometheus controller-manager metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = controller-manager # how often to collect prometheus metrics interval = 60s # prometheus endpoint endpoint = http://127.0.0.1:10252/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = # client certificate for authentication clientCertPath = # Allow invalid SSL server certificate insecure = false # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = [input.prometheus::etcd] # disable prometheus etcd metrics disabled = false # override type type = kubernetes_prometheus # specify Splunk index index = # override host host = ${KUBERNETES_NODENAME} # override source source = etcd # how often to collect prometheus metricd interval = 60s # prometheus endpoint endpoint.http = http://:2379/metrics endpoint.https = https://:2379/metrics # token for "Authorization: Bearer $(cat tokenPath)" tokenPath = # server certificate for certificate validation certPath = /rootfs/etc/kubernetes/pki/etcd/ca.crt # client certificate for authentication clientCertPath = /rootfs/etc/kubernetes/pki/apiserver-etcd-client.crt clientKeyPath = /rootfs/etc/kubernetes/pki/apiserver-etcd-client.key # Allow invalid SSL server certificate insecure = true # include metrics help with the events includeHelp = false # set output (splunk or devnull, default is [general]defaultOutput) output = 004-addon.conf: | [general] # addons can be run in parallel with agents addon = true [input.kubernetes_events] # disable events disabled = false # override type type = kubernetes_events # specify Splunk index index = # (obsolete, depends on kubernetes timeout) # Set the timeout for how long request to watch events going to hang reading. # eventsWatchTimeout = 30m # (obsolete, depends on kubernetes timeout) # Ignore events last seen later that this duration. # eventsTTL = 12h # set output (splunk or devnull, default is [general]defaultOutput) output = [input.kubernetes_watch::pods] # disable events disabled = false # Set the timeout for how often watch request should refresh the whole list refresh = 10m apiVersion = v1 kind = pod namespace = # override type type = kubernetes_objects # specify Splunk index index = # set output (splunk or devnull, default is [general]defaultOutput) output = collectorforkubernetes-cr.yaml000066400000000000000000000021211373430133400345510ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: collectorforkubernetes name: collectorforkubernetes rules: - apiGroups: ['extensions'] resources: ['podsecuritypolicies'] verbs: ['use'] resourceNames: - privileged - apiGroups: - "" - apps - batch - extensions - monitoring.coreos.com - etcd.database.coreos.com - vault.security.coreos.com - certmanager.k8s.io - kafka.strimzi.io - org.eclipse.che - dynatrace.com - nats.io - syndesis.io - collectord.io resources: - alertmanagers - cronjobs - daemonsets - deployments - endpoints - events - jobs - namespaces - nodes - nodes/metrics - nodes/proxy - pods - prometheuses - replicasets - replicationcontrollers - scheduledjobs - services - statefulsets - vaultservices - etcdclusters - challenges - kafkas - checlusters - oneagents - natsclusters - syndesises - persistentvolumeclaims - configurations verbs: - get - list - watch - nonResourceURLs: - /metrics verbs: - get apiGroups: [] resources: [] collectorforkubernetes-crb.yaml000066400000000000000000000006051373430133400347200ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: collectorforkubernetes name: collectorforkubernetes namespace: collectorforkubernetes roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: collectorforkubernetes subjects: - kind: ServiceAccount name: collectorforkubernetes namespace: collectorforkubernetes collectorforkubernetes-critical-priorityclass.yaml000066400000000000000000000001711373430133400406470ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: collectorforkubernetes-critical value: 1000000000 collectorforkubernetes-daemonset.yaml000066400000000000000000000123521373430133400361330ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: apps/v1 kind: DaemonSet metadata: name: collectorforkubernetes namespace: collectorforkubernetes labels: app: collectorforkubernetes spec: # Default updateStrategy is OnDelete. For collector RollingUpdate is suitable # When you update configuration updateStrategy: type: RollingUpdate selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes labels: daemon: collectorforkubernetes spec: priorityClassName: collectorforkubernetes-critical dnsPolicy: ClusterFirstWithHostNet hostNetwork: true serviceAccountName: collectorforkubernetes # We run this DaemonSet only for Non-Masters affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/master operator: DoesNotExist tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforkubernetes # Collector version image: docker.io/outcoldsolutions/collectorforkubernetes:5.14.284 imagePullPolicy: Always securityContext: runAsUser: 0 privileged: true # Define your resources if you need. Defaults should be fine for most. # You can lower or increase based on your hosts. resources: limits: cpu: 2 memory: 512Mi requests: cpu: 200m memory: 192Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name volumeMounts: # We store state in /data folder (file positions) - name: collectorforkubernetes-state mountPath: /data # Configuration file deployed with ConfigMap - name: collectorforkubernetes-config mountPath: /config/ readOnly: true # Cgroup filesystem to get metrics - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true # Proc filesystem to get metrics - name: proc mountPath: /rootfs/proc readOnly: true # Location of docker root (for container logs and metadata) - name: docker-root mountPath: /rootfs/var/lib/docker/ readOnly: true mountPropagation: HostToContainer # Docker socket - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true # CRI-O socket (if using CRI-O runtime) - name: crio-unix-socket mountPath: /rootfs/var/run/crio/ readOnly: true # Containerd socket (if using containerd runtime) - name: containerd-unix-socket mountPath: /rootfs/var/run/containerd/ readOnly: true # Host logs location (including CRI-O logs) - name: logs mountPath: /rootfs/var/log/ readOnly: true - name: run-logs mountPath: /rootfs/run/log/ readOnly: true # Application logs - name: volumes-root mountPath: /rootfs/var/lib/kubelet/ readOnly: true mountPropagation: HostToContainer # correct timezone - name: localtime mountPath: /etc/localtime readOnly: true volumes: # We store state directly on host, change this location, if # your persistent volume is somewhere else - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ # Location of docker root (for container logs and metadata) - name: docker-root hostPath: path: /var/lib/docker/ # Location of cgroups file system - name: cgroup hostPath: path: /sys/fs/cgroup # Location of proc file system - name: proc hostPath: path: /proc # Host logs location (including CRI-O logs) - name: logs hostPath: path: /var/log - name: run-logs hostPath: path: /run/log # Docker socket - name: docker-unix-socket hostPath: path: /var/run/docker.sock # CRI-O socket (if using CRI-O runtime) - name: crio-unix-socket hostPath: path: /var/run/crio/ # containerd socket (if using containerd runtime) - name: containerd-unix-socket hostPath: path: /var/run/containerd/ # Location for kubelet mounts, to autodiscover application logs - name: volumes-root hostPath: path: /var/lib/kubelet/ # correct timezone - name: localtime hostPath: path: /etc/localtime # configuration from ConfigMap - name: collectorforkubernetes-config configMap: name: collectorforkubernetes items: - key: 001-general.conf path: 001-general.conf - key: 002-daemonset.conf path: 002-daemonset.conf collectorforkubernetes-master-daemonset.yaml000066400000000000000000000102141373430133400374170ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: apps/v1 kind: DaemonSet metadata: name: collectorforkubernetes-master namespace: collectorforkubernetes labels: app: collectorforkubernetes spec: updateStrategy: type: RollingUpdate selector: matchLabels: daemon: collectorforkubernetes template: metadata: name: collectorforkubernetes-master labels: daemon: collectorforkubernetes spec: priorityClassName: collectorforkubernetes-critical dnsPolicy: ClusterFirstWithHostNet hostNetwork: true serviceAccountName: collectorforkubernetes affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/master operator: Exists tolerations: - operator: "Exists" effect: "NoSchedule" - operator: "Exists" effect: "NoExecute" containers: - name: collectorforkubernetes image: docker.io/outcoldsolutions/collectorforkubernetes:5.14.284 imagePullPolicy: Always securityContext: runAsUser: 0 privileged: true resources: limits: cpu: 2 memory: 512Mi requests: cpu: 200m memory: 192Mi env: - name: KUBERNETES_NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name volumeMounts: - name: collectorforkubernetes-state mountPath: /data - name: collectorforkubernetes-config mountPath: /config/ readOnly: true - name: cgroup mountPath: /rootfs/sys/fs/cgroup readOnly: true - name: proc mountPath: /rootfs/proc readOnly: true - name: docker-logs mountPath: /rootfs/var/lib/docker/ readOnly: true mountPropagation: HostToContainer - name: docker-unix-socket mountPath: /rootfs/var/run/docker.sock readOnly: true - name: crio-unix-socket mountPath: /rootfs/var/run/crio/ readOnly: true - name: containerd-unix-socket mountPath: /rootfs/var/run/containerd/ readOnly: true - name: logs mountPath: /rootfs/var/log/ readOnly: true - name: run-logs mountPath: /rootfs/run/log/ readOnly: true - name: k8s-certs mountPath: /rootfs/etc/kubernetes/pki/ readOnly: true - name: kubelet-root mountPath: /rootfs/var/lib/kubelet/ readOnly: true mountPropagation: HostToContainer - name: localtime mountPath: /etc/localtime readOnly: true volumes: - name: collectorforkubernetes-state hostPath: path: /var/lib/collectorforkubernetes/data/ - name: docker-logs hostPath: path: /var/lib/docker/ - name: cgroup hostPath: path: /sys/fs/cgroup - name: proc hostPath: path: /proc - name: logs hostPath: path: /var/log - name: run-logs hostPath: path: /run/log - name: docker-unix-socket hostPath: path: /var/run/docker.sock - name: crio-unix-socket hostPath: path: /var/run/crio/ - name: containerd-unix-socket hostPath: path: /var/run/containerd/ - name: k8s-certs hostPath: path: /etc/kubernetes/pki/ - name: kubelet-root hostPath: path: /var/lib/kubelet/ - name: localtime hostPath: path: /etc/localtime - name: collectorforkubernetes-config configMap: name: collectorforkubernetes items: - key: 001-general.conf path: 001-general.conf - key: 002-daemonset.conf path: 002-daemonset.conf - key: 003-daemonset-master.conf path: 003-daemonset-master.conf collectorforkubernetes-namespace.yaml000066400000000000000000000001621373430133400361040ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: v1 kind: Namespace metadata: labels: app: collectorforkubernetes name: collectorforkubernetes collectorforkubernetes-sa.yaml000066400000000000000000000002331373430133400345520ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: v1 kind: ServiceAccount metadata: labels: app: collectorforkubernetes name: collectorforkubernetes namespace: collectorforkubernetes configurations.collectord.io-crd.yaml000066400000000000000000000004771373430133400357350ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/collectorforkubernetes.yaml.goldenapiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: configurations.collectord.io spec: group: collectord.io versions: - name: v1 served: true storage: true scope: Cluster names: plural: configurations singular: configuration kind: Configuration kubernetes-split-yaml-0.3.0/testdata/simple.yaml000066400000000000000000000003651373430133400217450ustar00rootroot00000000000000apiVersion: v1 kind: Namespace metadata: name: simple --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: simple --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: simple --- kubernetes-split-yaml-0.3.0/testdata/simple.yaml.golden/000077500000000000000000000000001373430133400232655ustar00rootroot00000000000000kubernetes-split-yaml-0.3.0/testdata/simple.yaml.golden/simple-cr.yaml000066400000000000000000000001241373430133400260410ustar00rootroot00000000000000apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: simple kubernetes-split-yaml-0.3.0/testdata/simple.yaml.golden/simple-crb.yaml000066400000000000000000000001331373430133400262030ustar00rootroot00000000000000apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: simple kubernetes-split-yaml-0.3.0/testdata/simple.yaml.golden/simple-namespace.yaml000066400000000000000000000000701373430133400273710ustar00rootroot00000000000000apiVersion: v1 kind: Namespace metadata: name: simple kubernetes-split-yaml-0.3.0/yaml.go000066400000000000000000000012771373430133400172530ustar00rootroot00000000000000package main import ( "fmt" "strings" "gopkg.in/yaml.v2" ) func getYamlInfo(yamlContent string) (*KubernetesAPI, error) { // Start by removing all lines with templating in to create sane yaml cleanYaml := "" for _, line := range strings.Split(yamlContent, "\n") { if !strings.Contains(line, "{{") { cleanYaml += line + "\n" } } var m KubernetesAPI err := yaml.Unmarshal([]byte(cleanYaml), &m) if err != nil { return nil, fmt.Errorf("Could not unmarshal: %v \n---\n%v", err, yamlContent) } if m.Kind == "" { return nil, fmt.Errorf("yaml file with kind missing") } else if m.Metadata.Name == "" { return nil, fmt.Errorf("yaml file with name missing") } return &m, nil } kubernetes-split-yaml-0.3.0/yaml_test.go000066400000000000000000000036241373430133400203100ustar00rootroot00000000000000package main import ( "strings" "testing" ) func Test_getYamlInfo(t *testing.T) { type args struct { yamlContent string } tests := []struct { name string args args wantKind string wantName string wantNamespace string wantErr bool wantErrMsg string }{ { name: "Error on empty file", args: args{ yamlContent: "", }, wantKind: "", wantName: "", wantErr: true, wantErrMsg: "yaml file with kind missing", }, { name: "Error on non-yaml file", args: args{ yamlContent: "Non-yaml file....?", }, wantErr: true, wantErrMsg: "Could not unmarshal", }, { name: "Error on missing kind", args: args{ yamlContent: ` apiVersion: v1 metadata: name: simple `, }, wantErr: true, wantErrMsg: "yaml file with kind missing", }, { name: "Error on missing name", args: args{ yamlContent: `kind: Deployment`, }, wantErr: true, wantErrMsg: "yaml file with name missing", }, { name: "Can handle simple file", args: args{ yamlContent: ` apiVersion: v1 kind: Pod metadata: name: simple namespace: foo `, }, wantName: "simple", wantNamespace: "foo", wantKind: "Pod", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := getYamlInfo(tt.args.yamlContent) if (err != nil) != tt.wantErr { t.Errorf("getYamlInfo() error = %v, wantErr %v", err, tt.wantErr) return } if tt.wantErr && !strings.HasPrefix(err.Error(), tt.wantErrMsg) { t.Errorf("getYamlInfo() error = %v, wantErr %v", err.Error(), tt.wantErrMsg) return } if tt.wantErr { return } if got.Kind != tt.wantKind { t.Errorf("getYamlInfo() got = %v, want %v", got.Kind, tt.wantKind) } if got.Metadata.Name != tt.wantName { t.Errorf("getYamlInfo() got = %v, want %v", got.Metadata.Name, tt.wantName) } }) } }