pax_global_header00006660000000000000000000000064136237714600014523gustar00rootroot0000000000000052 comment=527cbcd6a7f966cf28a27a412ebf1374f548ac7a prometheus-elasticsearch-exporter-1.1.0+ds/000077500000000000000000000000001362377146000207755ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/.gitignore000066400000000000000000000000711362377146000227630ustar00rootroot00000000000000.build elasticsearch_exporter *.tar.gz *-stamp .tarballs prometheus-elasticsearch-exporter-1.1.0+ds/.gometalinter.json000066400000000000000000000003701362377146000244400ustar00rootroot00000000000000{ "Cyclo": 40, "Deadline": "6m", "EnableGC": true, "Exclude": ["TLS InsecureSkipVerify set true.", "Potential file inclusion via variable", "Errors unhandled.,LOW,HIGH"], "Sort": ["linter", "severity", "path", "line"], "Vendor": true } prometheus-elasticsearch-exporter-1.1.0+ds/.promu.yml000066400000000000000000000025151362377146000227430ustar00rootroot00000000000000go: cgo: false repository: path: github.com/justwatchcom/elasticsearch_exporter build: flags: -a -tags netgo ldflags: | -s -X {{repoPath}}/vendor/github.com/prometheus/common/version.Version={{.Version}} -X {{repoPath}}/vendor/github.com/prometheus/common/version.Revision={{.Revision}} -X {{repoPath}}/vendor/github.com/prometheus/common/version.Branch={{.Branch}} -X {{repoPath}}/vendor/github.com/prometheus/common/version.BuildUser={{user}}@{{host}} -X {{repoPath}}/vendor/github.com/prometheus/common/version.BuildDate={{date "20060102-15:04:05"}} tarball: files: - LICENSE - README.md - CHANGELOG.md - examples/grafana/dashboard.json - examples/kubernetes/deployment.yml - examples/prometheus/elasticsearch.rules crossbuild: platforms: - linux/amd64 - linux/386 - darwin/amd64 - darwin/386 - windows/amd64 - windows/386 - freebsd/amd64 - freebsd/386 - openbsd/amd64 - openbsd/386 - netbsd/amd64 - netbsd/386 - dragonfly/amd64 - linux/arm - linux/arm64 - freebsd/arm - openbsd/arm - netbsd/arm - linux/ppc64 - linux/ppc64le - linux/mips64 - linux/mips64le prometheus-elasticsearch-exporter-1.1.0+ds/.travis.yml000066400000000000000000000002611362377146000231050ustar00rootroot00000000000000language: go go: - 1.10.x - 1.11.x - tip script: - make style - make vet - make gometalinter - make build - make test matrix: allow_failures: - go: tip prometheus-elasticsearch-exporter-1.1.0+ds/CHANGELOG.md000066400000000000000000000077511362377146000226200ustar00rootroot00000000000000## 1.1.0 repeating the breaking changes introduced in 1.1.0rc1: * [BREAKING] uses the registered exporter port 9114 instead of 9118. If you need to stick to the old port, you can specify the listen port with --web.listen-address * [BREAKING] commandline flags are now POSIX flags with double dashes -- new changes in 1.1.0: * [FEATURE] add checksum promu command to Makefile * [FEATURE] add healthz handler * [BUGFIX] json parse error if the snapshot json contains failures (#269) * [BUGFIX] Remove credentials from URL in clusterinfo metrics * [FEATURE] Add indices_segment_term_vectors_memory_bytes_{primary,total} metrics * [FEATURE] Add indices_segments_{points,term_vectors,version_map}_memory_in_bytes metrics * [BUGFIX] Kubernetes yml file fixes * [FEATURE] Add index_stats_query_cache_caches_total metric * [FEATURE] Rename query_cache_cache_count metric to query_cache_cache_total * [BUGFIX] Change type for indices_query_cache_cache_count metric to counter * [BUGFIX]/ [BREAKING] Add _total prefix to indices_warmer_time_seconds metric * [FEATURE] Add indices_warmer_{time_seconds,total} metrics * [BUGFIX] exporter doesn't exit 1 if port is already in use (#241) * [BUGFIX] parse clusterinfo.build_date as string, not time.Time * [BUGFIX] Various Documentation Fixes * [FEATURE] add node_roles metric (#207) * [FEATURE] Extend nodes metrics. added indices.merges.current_size build fix: remove unnecessary conversion * [FEATURE] Extend nodes metrics. added overhead of circuit breakers * [BUGFIX] fix nodes metrics name indices.query_cache_miss_count, indices.request_cache_miss_count * [FEATURE] Extend nodes search metrics. added scroll_total, scroll_time * [FEATURE] Extend indices.indexing nodes metrics. added is_throttled, throttle_time * [FEATURE]/ [BUGFIX] #212 remove misleading metric ## 1.1.0rc1 * [BREAKING] uses the registered exporter port 9114 instead of 9118. If you need to stick to the old port, you can specify the listen port with --web.listen-address * [BREAKING] commandline flags are now POSIX flags with double dashes -- * [FEATURE] new collector for snapshot metrics * [FEATURE] added os memory stats metrics * [FEATURE] enable querying ES via proxy * [FEATURE] new collector for cluster settings * [FEATURE] new collector for indices settings * [FEATURE] cluster info collector. The collector periodically queries the / endpoints and provides the other collectors with a semi up-to-date cluster label * * [FEATURE]/ [BUGFIX] grafana dashboard improvements and fixes * [BUGFIX] Fixed createTLSConfig function. Return full tls configuration when ca, crt, key and insecure flag are set * * [INTERNAL] added code linting to build pipeline ## 1.0.4rc1 * [DOCUMENTATION] documentation updates * [FEATURE] add more index metrics * [FEATURE] add filesystem metrics * [FEATURE] add jvm buffer pool metrics * [FEATURE] add support for using the exporter behind reverse proxy (URL-prefixing) * [ENHANCEMENT] add linting to build chain and make project lint clean ## 1.0.3rc1 * [BUGFIX] update prometheus alerting rule example to v2 format * [ENHANCEMENT] Add formatting option for logger * [ENHANCEMENT] Add shard-level document count (can be toggled out) * [ENHANCEMENT] Add OS CPU usage metric * [ENHANCEMENT] Add skip-ssl-verify option * [ENHANCEMENT] Add node-level current merge metrics ## 1.0.2 / 2018-01-09 * [ENHANCEMENT] Add index metrics [#85] [#116] [#118] * [ENHANCEMENT] Add cache metrics [#88] * [ENHANCEMENT] Add documentation for the example dashboard [#84] * [ENHANCEMNET] Expose load averages [#113] * [BUGFIX] Fix role detection [#105] [#110] * [BUGFIX] Fix indexing calls and time metrics [#83] ## 1.0.1 / 2017-07-24 * [ENHANCEMENT] Add exporter instrumentation [#78] * [BUGFIX] Exclude basic auth credentials from log [#71] * [BUGFIX] Fix missing node store size metric ## 1.0.0 / 2017-07-03 * [ENHANCEMENT] Rewrite the codebase to reduce redundancy and improve extensibility [#65] * [ENHANCEMENT] Add examples for Grafana and Prometheus [#66] * [BREAKING] Removed several duplicate or redundant metrics [#65] prometheus-elasticsearch-exporter-1.1.0+ds/Dockerfile000066400000000000000000000007661362377146000230000ustar00rootroot00000000000000FROM quay.io/prometheus/golang-builder as builder ADD . /go/src/github.com/justwatchcom/elasticsearch_exporter WORKDIR /go/src/github.com/justwatchcom/elasticsearch_exporter RUN make FROM quay.io/prometheus/busybox:latest MAINTAINER The Prometheus Authors COPY --from=builder /go/src/github.com/justwatchcom/elasticsearch_exporter/elasticsearch_exporter /bin/elasticsearch_exporter EXPOSE 9114 ENTRYPOINT [ "/bin/elasticsearch_exporter" ] prometheus-elasticsearch-exporter-1.1.0+ds/LICENSE000066400000000000000000000261351362377146000220110ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. prometheus-elasticsearch-exporter-1.1.0+ds/Makefile000066400000000000000000000045671362377146000224510ustar00rootroot00000000000000# Copyright 2016 The Prometheus Authors # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. GO := GO15VENDOREXPERIMENT=1 go PROMU := $(GOPATH)/bin/promu GOLINTER ?= $(GOPATH)/bin/gometalinter pkgs = $(shell $(GO) list ./... | grep -v /vendor/) PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_NAME ?= elasticsearch-exporter DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) all: format build test style: @echo ">> checking code style" @! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^' test: @echo ">> running tests" @$(GO) test -short $(pkgs) format: @echo ">> formatting code" @$(GO) fmt $(pkgs) vet: @echo ">> vetting code" @$(GO) vet $(pkgs) build: promu @echo ">> building binaries" @$(PROMU) build --prefix $(PREFIX) crossbuild: promu @echo ">> cross-building binaries" @$(PROMU) crossbuild tarball: promu @echo ">> building release tarball" @$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR) tarballs: promu @echo ">> building release tarballs" @$(PROMU) crossbuild tarballs @echo ">> calculating release checksums" @$(PROMU) checksum $(BIN_DIR)/.tarballs docker: @echo ">> building docker image" @docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" . promu: @GOOS=$(shell uname -s | tr A-Z a-z) \ GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) \ $(GO) get -u github.com/prometheus/promu gometalinter: $(GOLINTER) @echo ">> linting code" @$(GOLINTER) --install > /dev/null @$(GOLINTER) --config=./.gometalinter.json ./... $(GOPATH)/bin/gometalinter lint: @GOOS=$(shell uname -s | tr A-Z a-z) \ GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) \ $(GO) get -u github.com/alecthomas/gometalinter .PHONY: all style format build test vet tarball docker promu $(GOPATH)/bin/gometalinter lint prometheus-elasticsearch-exporter-1.1.0+ds/README.md000066400000000000000000000534571362377146000222720ustar00rootroot00000000000000# Elasticsearch Exporter [![Build Status](https://travis-ci.org/justwatchcom/elasticsearch_exporter.svg?branch=master)](https://travis-ci.org/justwatchcom/elasticsearch_exporter) [![Docker Pulls](https://img.shields.io/docker/pulls/justwatch/elasticsearch_exporter.svg?maxAge=604800)](https://hub.docker.com/r/justwatch/elasticsearch_exporter) [![Go Report Card](https://goreportcard.com/badge/github.com/justwatchcom/elasticsearch_exporter)](https://goreportcard.com/report/github.com/justwatchcom/elasticsearch_exporter) Prometheus exporter for various metrics about ElasticSearch, written in Go. ### Installation For pre-built binaries please take a look at the releases. https://github.com/justwatchcom/elasticsearch_exporter/releases #### Docker ```bash docker pull justwatch/elasticsearch_exporter:1.1.0 docker run --rm -p 9114:9114 justwatch/elasticsearch_exporter:1.1.0 ``` Example `docker-compose.yml`: ```yaml elasticsearch_exporter: image: justwatch/elasticsearch_exporter:1.1.0 command: - '-es.uri=http://elasticsearch:9200' restart: always ports: - "127.0.0.1:9114:9114" ``` #### Kubernetes You can find a helm chart in the stable charts repository at https://github.com/kubernetes/charts/tree/master/stable/elasticsearch-exporter. ### Configuration **NOTE:** The exporter fetches information from an ElasticSearch cluster on every scrape, therefore having a too short scrape interval can impose load on ES master nodes, particularly if you run with `-es.all` and `-es.indices`. We suggest you measure how long fetching `/_nodes/stats` and `/_all/_stats` takes for your ES cluster to determine whether your scraping interval is too short. As a last resort, you can scrape this exporter using a dedicated job with its own scraping interval. Below is the command line options summary: ```bash elasticsearch_exporter --help ``` | Argument | Introduced in Version | Description | Default | | -------- | --------------------- | ----------- | ----------- | | es.uri | 1.0.2 | Address (host and port) of the Elasticsearch node we should connect to. This could be a local node (`localhost:9200`, for instance), or the address of a remote Elasticsearch server. When basic auth is needed, specify as: `://:@:`. E.G., `http://admin:pass@localhost:9200`. | http://localhost:9200 | | es.all | 1.0.2 | If true, query stats for all nodes in the cluster, rather than just the node we connect to. | false | | es.cluster_settings | 1.1.0rc1 | If true, query stats for cluster settings. | false | | es.indices | 1.0.2 | If true, query stats for all indices in the cluster. | false | | es.indices_settings | 1.0.4rc1 | If true, query settings stats for all indices in the cluster. | false | | es.shards | 1.0.3rc1 | If true, query stats for all indices in the cluster, including shard-level stats (implies `es.indices=true`). | false | | es.snapshots | 1.0.4rc1 | If true, query stats for the cluster snapshots. | false | | es.timeout | 1.0.2 | Timeout for trying to get stats from Elasticsearch. (ex: 20s) | 5s | | es.ca | 1.0.2 | Path to PEM file that contains trusted Certificate Authorities for the Elasticsearch connection. | | | es.client-private-key | 1.0.2 | Path to PEM file that contains the private key for client auth when connecting to Elasticsearch. | | | es.client-cert | 1.0.2 | Path to PEM file that contains the corresponding cert for the private key to connect to Elasticsearch. | | | es.clusterinfo.interval | 1.1.0rc1 | Cluster info update interval for the cluster label | 5m | | es.ssl-skip-verify | 1.0.4rc1 | Skip SSL verification when connecting to Elasticsearch. | false | | web.listen-address | 1.0.2 | Address to listen on for web interface and telemetry. | :9114 | | web.telemetry-path | 1.0.2 | Path under which to expose metrics. | /metrics | | version | 1.0.2 | Show version info on stdout and exit. | | Commandline parameters start with a single `-` for versions less than `1.1.0rc1`. For versions greater than `1.1.0rc1`, commandline parameters are specified with `--`. Also, all commandline parameters can be provided as environment variables. The environment variable name is derived from the parameter name by replacing `.` and `-` with `_` and upper-casing the parameter name. ### Metrics |Name |Type |Cardinality |Help |---- |---- |----------- |---- | elasticsearch_breakers_estimated_size_bytes | gauge | 4 | Estimated size in bytes of breaker | elasticsearch_breakers_limit_size_bytes | gauge | 4 | Limit size in bytes for breaker | elasticsearch_breakers_tripped | counter | 4 | tripped for breaker | elasticsearch_cluster_health_active_primary_shards | gauge | 1 | The number of primary shards in your cluster. This is an aggregate total across all indices. | elasticsearch_cluster_health_active_shards | gauge | 1 | Aggregate total of all shards across all indices, which includes replica shards. | elasticsearch_cluster_health_delayed_unassigned_shards | gauge | 1 | Shards delayed to reduce reallocation overhead | elasticsearch_cluster_health_initializing_shards | gauge | 1 | Count of shards that are being freshly created. | elasticsearch_cluster_health_number_of_data_nodes | gauge | 1 | Number of data nodes in the cluster. | elasticsearch_cluster_health_number_of_in_flight_fetch | gauge | 1 | The number of ongoing shard info requests. | elasticsearch_cluster_health_number_of_nodes | gauge | 1 | Number of nodes in the cluster. | elasticsearch_cluster_health_number_of_pending_tasks | gauge | 1 | Cluster level changes which have not yet been executed | elasticsearch_cluster_health_task_max_waiting_in_queue_millis | gauge | 1 | Max time in millis that a task is waiting in queue. | elasticsearch_cluster_health_relocating_shards | gauge | 1 | The number of shards that are currently moving from one node to another node. | elasticsearch_cluster_health_status | gauge | 3 | Whether all primary and replica shards are allocated. | elasticsearch_cluster_health_timed_out | gauge | 1 | Number of cluster health checks timed out | elasticsearch_cluster_health_unassigned_shards | gauge | 1 | The number of shards that exist in the cluster state, but cannot be found in the cluster itself. | elasticsearch_filesystem_data_available_bytes | gauge | 1 | Available space on block device in bytes | elasticsearch_filesystem_data_free_bytes | gauge | 1 | Free space on block device in bytes | elasticsearch_filesystem_data_size_bytes | gauge | 1 | Size of block device in bytes | elasticsearch_filesystem_io_stats_device_operations_count | gauge | 1 | Count of disk operations | elasticsearch_filesystem_io_stats_device_read_operations_count | gauge | 1 | Count of disk read operations | elasticsearch_filesystem_io_stats_device_write_operations_count | gauge | 1 | Count of disk write operations | elasticsearch_filesystem_io_stats_device_read_size_kilobytes_sum | gauge | 1 | Total kilobytes read from disk | elasticsearch_filesystem_io_stats_device_write_size_kilobytes_sum | gauge | 1 | Total kilobytes written to disk | elasticsearch_indices_docs | gauge | 1 | Count of documents on this node | elasticsearch_indices_docs_deleted | gauge | 1 | Count of deleted documents on this node | elasticsearch_indices_docs_primary | gauge | | Count of documents with only primary shards on all nodes | elasticsearch_indices_fielddata_evictions | counter | 1 | Evictions from field data | elasticsearch_indices_fielddata_memory_size_bytes | gauge | 1 | Field data cache memory usage in bytes | elasticsearch_indices_filter_cache_evictions | counter | 1 | Evictions from filter cache | elasticsearch_indices_filter_cache_memory_size_bytes | gauge | 1 | Filter cache memory usage in bytes | elasticsearch_indices_flush_time_seconds | counter | 1 | Cumulative flush time in seconds | elasticsearch_indices_flush_total | counter | 1 | Total flushes | elasticsearch_indices_get_exists_time_seconds | counter | 1 | Total time get exists in seconds | elasticsearch_indices_get_exists_total | counter | 1 | Total get exists operations | elasticsearch_indices_get_missing_time_seconds | counter | 1 | Total time of get missing in seconds | elasticsearch_indices_get_missing_total | counter | 1 | Total get missing | elasticsearch_indices_get_time_seconds | counter | 1 | Total get time in seconds | elasticsearch_indices_get_total | counter | 1 | Total get | elasticsearch_indices_indexing_delete_time_seconds_total | counter | 1 | Total time indexing delete in seconds | elasticsearch_indices_indexing_delete_total | counter | 1 | Total indexing deletes | elasticsearch_indices_indexing_index_time_seconds_total | counter | 1 | Cumulative index time in seconds | elasticsearch_indices_indexing_index_total | counter | 1 | Total index calls | elasticsearch_indices_merges_docs_total | counter | 1 | Cumulative docs merged | elasticsearch_indices_merges_total | counter | 1 | Total merges | elasticsearch_indices_merges_total_size_bytes_total | counter | 1 | Total merge size in bytes | elasticsearch_indices_merges_total_time_seconds_total | counter | 1 | Total time spent merging in seconds | elasticsearch_indices_query_cache_cache_total | counter | 1 | Count of query cache | elasticsearch_indices_query_cache_cache_size | gauge | 1 | Size of query cache | elasticsearch_indices_query_cache_count | counter | 2 | Count of query cache hit/miss | elasticsearch_indices_query_cache_evictions | counter | 1 | Evictions from query cache | elasticsearch_indices_query_cache_memory_size_bytes | gauge | 1 | Query cache memory usage in bytes | elasticsearch_indices_query_cache_total | counter | 1 | Size of query cache total | elasticsearch_indices_refresh_time_seconds_total | counter | 1 | Total time spent refreshing in seconds | elasticsearch_indices_refresh_total | counter | 1 | Total refreshes | elasticsearch_indices_request_cache_count | counter | 2 | Count of request cache hit/miss | elasticsearch_indices_request_cache_evictions | counter | 1 | Evictions from request cache | elasticsearch_indices_request_cache_memory_size_bytes | gauge | 1 | Request cache memory usage in bytes | elasticsearch_indices_search_fetch_time_seconds | counter | 1 | Total search fetch time in seconds | elasticsearch_indices_search_fetch_total | counter | 1 | Total number of fetches | elasticsearch_indices_search_query_time_seconds | counter | 1 | Total search query time in seconds | elasticsearch_indices_search_query_total | counter | 1 | Total number of queries | elasticsearch_indices_segments_count | gauge | 1 | Count of index segments on this node | elasticsearch_indices_segments_memory_bytes | gauge | 1 | Current memory size of segments in bytes | elasticsearch_indices_settings_stats_read_only_indices | gauge | 1 | Count of indices that have read_only_allow_delete=true | elasticsearch_indices_shards_docs | gauge | 3 | Count of documents on this shard | elasticsearch_indices_shards_docs_deleted | gauge | 3 | Count of deleted documents on each shard | elasticsearch_indices_store_size_bytes | gauge | 1 | Current size of stored index data in bytes | elasticsearch_indices_store_size_bytes_primary | gauge | | Current size of stored index data in bytes with only primary shards on all nodes | elasticsearch_indices_store_size_bytes_total | gauge | | Current size of stored index data in bytes with all shards on all nodes | elasticsearch_indices_store_throttle_time_seconds_total | counter | 1 | Throttle time for index store in seconds | elasticsearch_indices_translog_operations | counter | 1 | Total translog operations | elasticsearch_indices_translog_size_in_bytes | counter | 1 | Total translog size in bytes | elasticsearch_indices_warmer_time_seconds_total | counter | 1 | Total warmer time in seconds | elasticsearch_indices_warmer_total | counter | 1 | Total warmer count | elasticsearch_jvm_gc_collection_seconds_count | counter | 2 | Count of JVM GC runs | elasticsearch_jvm_gc_collection_seconds_sum | counter | 2 | GC run time in seconds | elasticsearch_jvm_memory_committed_bytes | gauge | 2 | JVM memory currently committed by area | elasticsearch_jvm_memory_max_bytes | gauge | 1 | JVM memory max | elasticsearch_jvm_memory_used_bytes | gauge | 2 | JVM memory currently used by area | elasticsearch_jvm_memory_pool_used_bytes | gauge | 3 | JVM memory currently used by pool | elasticsearch_jvm_memory_pool_max_bytes | counter | 3 | JVM memory max by pool | elasticsearch_jvm_memory_pool_peak_used_bytes | counter | 3 | JVM memory peak used by pool | elasticsearch_jvm_memory_pool_peak_max_bytes | counter | 3 | JVM memory peak max by pool | elasticsearch_os_cpu_percent | gauge | 1 | Percent CPU used by the OS | elasticsearch_os_load1 | gauge | 1 | Shortterm load average | elasticsearch_os_load5 | gauge | 1 | Midterm load average | elasticsearch_os_load15 | gauge | 1 | Longterm load average | elasticsearch_process_cpu_percent | gauge | 1 | Percent CPU used by process | elasticsearch_process_cpu_time_seconds_sum | counter | 3 | Process CPU time in seconds | elasticsearch_process_mem_resident_size_bytes | gauge | 1 | Resident memory in use by process in bytes | elasticsearch_process_mem_share_size_bytes | gauge | 1 | Shared memory in use by process in bytes | elasticsearch_process_mem_virtual_size_bytes | gauge | 1 | Total virtual memory used in bytes | elasticsearch_process_open_files_count | gauge | 1 | Open file descriptors | elasticsearch_snapshot_stats_number_of_snapshots | gauge | 1 | Total number of snapshots | elasticsearch_snapshot_stats_oldest_snapshot_timestamp | gauge | 1 | Oldest snapshot timestamp | elasticsearch_snapshot_stats_snapshot_start_time_timestamp | gauge | 1 | Last snapshot start timestamp | elasticsearch_snapshot_stats_snapshot_end_time_timestamp | gauge | 1 | Last snapshot end timestamp | elasticsearch_snapshot_stats_snapshot_number_of_failures | gauge | 1 | Last snapshot number of failures | elasticsearch_snapshot_stats_snapshot_number_of_indices | gauge | 1 | Last snapshot number of indices | elasticsearch_snapshot_stats_snapshot_failed_shards | gauge | 1 | Last snapshot failed shards | elasticsearch_snapshot_stats_snapshot_successful_shards | gauge | 1 | Last snapshot successful shards | elasticsearch_snapshot_stats_snapshot_total_shards | gauge | 1 | Last snapshot total shard | elasticsearch_thread_pool_active_count | gauge | 14 | Thread Pool threads active | elasticsearch_thread_pool_completed_count | counter | 14 | Thread Pool operations completed | elasticsearch_thread_pool_largest_count | gauge | 14 | Thread Pool largest threads count | elasticsearch_thread_pool_queue_count | gauge | 14 | Thread Pool operations queued | elasticsearch_thread_pool_rejected_count | counter | 14 | Thread Pool operations rejected | elasticsearch_thread_pool_threads_count | gauge | 14 | Thread Pool current threads count | elasticsearch_transport_rx_packets_total | counter | 1 | Count of packets received | elasticsearch_transport_rx_size_bytes_total | counter | 1 | Total number of bytes received | elasticsearch_transport_tx_packets_total | counter | 1 | Count of packets sent | elasticsearch_transport_tx_size_bytes_total | counter | 1 | Total number of bytes sent | elasticsearch_clusterinfo_last_retrieval_success_ts | gauge | 1 | Timestamp of the last successful cluster info retrieval | elasticsearch_clusterinfo_up | gauge | 1 | Up metric for the cluster info collector | elasticsearch_clusterinfo_version_info | gauge | 6 | Constant metric with ES version information as labels ### Alerts & Recording Rules We provide examples for [Prometheus](http://prometheus.io) [alerts and recording rules](examples/prometheus/elasticsearch.rules) as well as an [Grafana](http://www.grafana.org) [Dashboard](examples/grafana/dashboard.json) and a [Kubernetes](http://kubernetes.io) [Deployment](examples/kubernetes/deployment.yml). The example dashboard needs the [node_exporter](https://github.com/prometheus/node_exporter) installed. In order to select the nodes that belong to the ElasticSearch cluster, we rely on a label `cluster`. Depending on your setup, it can derived from the platform metadata: For example on [GCE](https://cloud.google.com) ``` - source_labels: [__meta_gce_metadata_Cluster] separator: ; regex: (.*) target_label: cluster replacement: ${1} action: replace ``` Please refer to the [Prometheus SD documentation](https://prometheus.io/docs/operating/configuration/) to see which metadata labels can be used to create the `cluster` label. ## Credit & License `elasticsearch_exporter` is maintained by the nice folks from [JustWatch](https://www.justwatch.com/) and licensed under the terms of the Apache license. This package was originally created and maintained by [Eric Richardson](https://github.com/ewr), who transferred this repository to us in January 2017. Maintainers of this repository: * Christoph Oelmüller @zwopir Please refer to the Git commit log for a complete list of contributors. ## Contributing We welcome any contributions. Please fork the project on GitHub and open Pull Requests for any proposed changes. Please note that we will not merge any changes that encourage insecure behaviour. If in doubt please open an Issue first to discuss your proposal. prometheus-elasticsearch-exporter-1.1.0+ds/VERSION000066400000000000000000000000061362377146000220410ustar00rootroot000000000000001.1.0 prometheus-elasticsearch-exporter-1.1.0+ds/collector/000077500000000000000000000000001362377146000227635ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/collector/cluster_health.go000066400000000000000000000206011362377146000263170ustar00rootroot00000000000000package collector import ( "encoding/json" "fmt" "net/http" "net/url" "path" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) const ( namespace = "elasticsearch" ) var ( colors = []string{"green", "yellow", "red"} defaultClusterHealthLabels = []string{"cluster"} ) type clusterHealthMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(clusterHealth clusterHealthResponse) float64 } type clusterHealthStatusMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(clusterHealth clusterHealthResponse, color string) float64 Labels func(clusterName, color string) []string } // ClusterHealth type defines the collector struct type ClusterHealth struct { logger log.Logger client *http.Client url *url.URL up prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter metrics []*clusterHealthMetric statusMetric *clusterHealthStatusMetric } // NewClusterHealth returns a new Collector exposing ClusterHealth stats. func NewClusterHealth(logger log.Logger, client *http.Client, url *url.URL) *ClusterHealth { subsystem := "cluster_health" return &ClusterHealth{ logger: logger, client: client, url: url, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "up"), Help: "Was the last scrape of the ElasticSearch cluster health endpoint successful.", }), totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "total_scrapes"), Help: "Current total ElasticSearch cluster health scrapes.", }), jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "json_parse_failures"), Help: "Number of errors while parsing JSON.", }), metrics: []*clusterHealthMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "active_primary_shards"), "The number of primary shards in your cluster. This is an aggregate total across all indices.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.ActivePrimaryShards) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "active_shards"), "Aggregate total of all shards across all indices, which includes replica shards.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.ActiveShards) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "delayed_unassigned_shards"), "Shards delayed to reduce reallocation overhead", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.DelayedUnassignedShards) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "initializing_shards"), "Count of shards that are being freshly created.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.InitializingShards) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "number_of_data_nodes"), "Number of data nodes in the cluster.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.NumberOfDataNodes) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "number_of_in_flight_fetch"), "The number of ongoing shard info requests.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.NumberOfInFlightFetch) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "task_max_waiting_in_queue_millis"), "Tasks max time waiting in queue.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.TaskMaxWaitingInQueueMillis) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "number_of_nodes"), "Number of nodes in the cluster.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.NumberOfNodes) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "number_of_pending_tasks"), "Cluster level changes which have not yet been executed", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.NumberOfPendingTasks) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "relocating_shards"), "The number of shards that are currently moving from one node to another node.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.RelocatingShards) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "unassigned_shards"), "The number of shards that exist in the cluster state, but cannot be found in the cluster itself.", defaultClusterHealthLabels, nil, ), Value: func(clusterHealth clusterHealthResponse) float64 { return float64(clusterHealth.UnassignedShards) }, }, }, statusMetric: &clusterHealthStatusMetric{ Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, subsystem, "status"), "Whether all primary and replica shards are allocated.", []string{"cluster", "color"}, nil, ), Value: func(clusterHealth clusterHealthResponse, color string) float64 { if clusterHealth.Status == color { return 1 } return 0 }, }, } } // Describe set Prometheus metrics descriptions. func (c *ClusterHealth) Describe(ch chan<- *prometheus.Desc) { for _, metric := range c.metrics { ch <- metric.Desc } ch <- c.statusMetric.Desc ch <- c.up.Desc() ch <- c.totalScrapes.Desc() ch <- c.jsonParseFailures.Desc() } func (c *ClusterHealth) fetchAndDecodeClusterHealth() (clusterHealthResponse, error) { var chr clusterHealthResponse u := *c.url u.Path = path.Join(u.Path, "/_cluster/health") res, err := c.client.Get(u.String()) if err != nil { return chr, fmt.Errorf("failed to get cluster health from %s://%s:%s%s: %s", u.Scheme, u.Hostname(), u.Port(), u.Path, err) } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(c.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(&chr); err != nil { c.jsonParseFailures.Inc() return chr, err } return chr, nil } // Collect collects ClusterHealth metrics. func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric) { var err error c.totalScrapes.Inc() defer func() { ch <- c.up ch <- c.totalScrapes ch <- c.jsonParseFailures }() clusterHealthResp, err := c.fetchAndDecodeClusterHealth() if err != nil { c.up.Set(0) _ = level.Warn(c.logger).Log( "msg", "failed to fetch and decode cluster health", "err", err, ) return } c.up.Set(1) for _, metric := range c.metrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(clusterHealthResp), clusterHealthResp.ClusterName, ) } for _, color := range colors { ch <- prometheus.MustNewConstMetric( c.statusMetric.Desc, c.statusMetric.Type, c.statusMetric.Value(clusterHealthResp, color), clusterHealthResp.ClusterName, color, ) } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/cluster_health_response.go000066400000000000000000000020271362377146000302370ustar00rootroot00000000000000package collector type clusterHealthResponse struct { ClusterName string `json:"cluster_name"` Status string `json:"status"` TimedOut bool `json:"timed_out"` NumberOfNodes int `json:"number_of_nodes"` NumberOfDataNodes int `json:"number_of_data_nodes"` ActivePrimaryShards int `json:"active_primary_shards"` ActiveShards int `json:"active_shards"` RelocatingShards int `json:"relocating_shards"` InitializingShards int `json:"initializing_shards"` UnassignedShards int `json:"unassigned_shards"` DelayedUnassignedShards int `json:"delayed_unassigned_shards"` NumberOfPendingTasks int `json:"number_of_pending_tasks"` NumberOfInFlightFetch int `json:"number_of_in_flight_fetch"` TaskMaxWaitingInQueueMillis int `json:"task_max_waiting_in_queue_millis"` ActiveShardsPercentAsNumber float64 `json:"active_shards_percent_as_number"` } prometheus-elasticsearch-exporter-1.1.0+ds/collector/cluster_health_test.go000066400000000000000000000047651362377146000273730ustar00rootroot00000000000000package collector import ( "fmt" "net/http" "net/http/httptest" "net/url" "testing" "github.com/go-kit/kit/log" ) func TestClusterHealth(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine // curl -XPUT http://localhost:9200/twitter // curl http://localhost:9200/_cluster/health tcs := map[string]string{ "1.7.6": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0}`, "2.4.5": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":12,"active_shards_percent_as_number":50.0}`, "5.4.2": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":12,"active_shards_percent_as_number":50.0}`, } for ver, out := range tcs { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, out) })) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("Failed to parse URL: %s", err) } c := NewClusterHealth(log.NewNopLogger(), http.DefaultClient, u) chr, err := c.fetchAndDecodeClusterHealth() if err != nil { t.Fatalf("Failed to fetch or decode cluster health: %s", err) } t.Logf("[%s] Cluster Health Response: %+v", ver, chr) if chr.ClusterName != "elasticsearch" { t.Errorf("Invalid cluster health response") } if chr.Status != "yellow" { t.Errorf("Invalid cluster status") } if chr.TimedOut { t.Errorf("Check didn't time out") } if chr.NumberOfNodes != 1 { t.Errorf("Wrong number of nodes") } if chr.NumberOfDataNodes != 1 { t.Errorf("Wrong number of data nodes") } if ver != "1.7.6" { if chr.TaskMaxWaitingInQueueMillis != 12 { t.Errorf("Wrong task max waiting time in millis") } } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/cluster_settings.go000066400000000000000000000076051362377146000267230ustar00rootroot00000000000000package collector import ( "encoding/json" "fmt" "net/http" "net/url" "path" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/imdario/mergo" "github.com/prometheus/client_golang/prometheus" ) // ClusterSettings information struct type ClusterSettings struct { logger log.Logger client *http.Client url *url.URL up prometheus.Gauge shardAllocationEnabled prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter } // NewClusterSettings defines Cluster Settings Prometheus metrics func NewClusterSettings(logger log.Logger, client *http.Client, url *url.URL) *ClusterSettings { return &ClusterSettings{ logger: logger, client: client, url: url, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "up"), Help: "Was the last scrape of the ElasticSearch cluster settings endpoint successful.", }), totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "total_scrapes"), Help: "Current total ElasticSearch cluster settings scrapes.", }), shardAllocationEnabled: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "shard_allocation_enabled"), Help: "Current mode of cluster wide shard routing allocation settings.", }), jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "clustersettings_stats", "json_parse_failures"), Help: "Number of errors while parsing JSON.", }), } } // Describe add Snapshots metrics descriptions func (cs *ClusterSettings) Describe(ch chan<- *prometheus.Desc) { ch <- cs.up.Desc() ch <- cs.totalScrapes.Desc() ch <- cs.shardAllocationEnabled.Desc() ch <- cs.jsonParseFailures.Desc() } func (cs *ClusterSettings) getAndParseURL(u *url.URL, data interface{}) error { res, err := cs.client.Get(u.String()) if err != nil { return fmt.Errorf("failed to get from %s://%s:%s%s: %s", u.Scheme, u.Hostname(), u.Port(), u.Path, err) } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(cs.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(data); err != nil { cs.jsonParseFailures.Inc() return err } return nil } func (cs *ClusterSettings) fetchAndDecodeClusterSettingsStats() (ClusterSettingsResponse, error) { u := *cs.url u.Path = path.Join(u.Path, "/_cluster/settings") q := u.Query() q.Set("include_defaults", "true") u.RawPath = q.Encode() var csfr ClusterSettingsFullResponse var csr ClusterSettingsResponse err := cs.getAndParseURL(&u, &csfr) if err != nil { return csr, err } err = mergo.Merge(&csr, csfr.Defaults, mergo.WithOverride) if err != nil { return csr, err } err = mergo.Merge(&csr, csfr.Persistent, mergo.WithOverride) if err != nil { return csr, err } err = mergo.Merge(&csr, csfr.Transient, mergo.WithOverride) return csr, err } // Collect gets cluster settings metric values func (cs *ClusterSettings) Collect(ch chan<- prometheus.Metric) { cs.totalScrapes.Inc() defer func() { ch <- cs.up ch <- cs.totalScrapes ch <- cs.jsonParseFailures ch <- cs.shardAllocationEnabled }() csr, err := cs.fetchAndDecodeClusterSettingsStats() if err != nil { cs.shardAllocationEnabled.Set(0) cs.up.Set(0) _ = level.Warn(cs.logger).Log( "msg", "failed to fetch and decode cluster settings stats", "err", err, ) return } cs.up.Set(1) shardAllocationMap := map[string]int{ "all": 0, "primaries": 1, "new_primaries": 2, "none": 3, } cs.shardAllocationEnabled.Set(float64(shardAllocationMap[csr.Cluster.Routing.Allocation.Enabled])) } prometheus-elasticsearch-exporter-1.1.0+ds/collector/cluster_settings_response.go000066400000000000000000000016141362377146000306330ustar00rootroot00000000000000package collector // ClusterSettingsFullResponse is a representation of a Elasticsearch Cluster Settings type ClusterSettingsFullResponse struct { Defaults ClusterSettingsResponse `json:"defaults"` Persistent ClusterSettingsResponse `json:"persistent"` Transient ClusterSettingsResponse `json:"transient"` } // ClusterSettingsResponse is a representation of a Elasticsearch Cluster Settings type ClusterSettingsResponse struct { Cluster Cluster `json:"cluster"` } // Cluster is a representation of a Elasticsearch Cluster Settings type Cluster struct { Routing Routing `json:"routing"` } // Routing is a representation of a Elasticsearch Cluster shard routing configuration type Routing struct { Allocation Allocation `json:"allocation"` } // Allocation is a representation of a Elasticsearch Cluster shard routing allocation settings type Allocation struct { Enabled string `json:"enable"` } prometheus-elasticsearch-exporter-1.1.0+ds/collector/cluster_settings_test.go000066400000000000000000000474211362377146000277620ustar00rootroot00000000000000package collector import ( "fmt" "net/http" "net/http/httptest" "net/url" "testing" "github.com/go-kit/kit/log" ) func TestClusterSettingsStats(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine // curl http://localhost:9200/_cluster/settings/?include_defaults=true tcs := map[string]string{ "5.4.2": `{"persistent":{},"transient":{},"defaults":{"cluster":{"routing":{"rebalance":{"enable":"ALL"},"allocation":{"node_concurrent_incoming_recoveries":"2","node_initial_primaries_recoveries":"4","same_shard":{"host":"false"},"total_shards_per_node":"-1","type":"balanced","disk":{"threshold_enabled":"true","watermark":{"low":"85%","high":"90%"},"include_relocations":"true","reroute_interval":"60s"},"awareness":{"attributes":""},"balance":{"index":"0.55","threshold":"1.0","shard":"0.45"},"enable":"ALL","node_concurrent_outgoing_recoveries":"2","allow_rebalance":"indices_all_active","cluster_concurrent_rebalance":"2","node_concurrent_recoveries":"2","snapshot":{"relocation_enabled":"false"}}},"indices":{"close":{"enable":"true"}},"nodes":{"reconnect_interval":"10s"},"blocks":{"read_only":"false"},"service":{"slow_task_logging_threshold":"30s"},"name":"elasticsearch","info":{"update":{"interval":"30s","timeout":"15s"}}},"logger":{"level":"INFO"},"bootstrap":{"ctrlhandler":"true","memory_lock":"false","system_call_filter":"true","seccomp":"true"},"processors":"4","network":{"host":["_local_"],"tcp":{"reuse_address":"true","connect_timeout":"30s","blocking":"false","blocking_server":"false","no_delay":"true","blocking_client":"false","keep_alive":"true","receive_buffer_size":"-1b","send_buffer_size":"-1b"},"bind_host":["_local_"],"server":"true","breaker":{"inflight_requests":{"limit":"100%","overhead":"1.0"}},"publish_host":["_local_"]},"pidfile":"","path":{"conf":"","scripts":"","logs":"/usr/share/elasticsearch/logs","shared_data":"","home":"/usr/share/elasticsearch"},"default":{"path":{"logs":"","conf":""}},"search":{"default_search_timeout":"-1","highlight":{"term_vector_multi_value":"true"},"low_level_cancellation":"false","keep_alive_interval":"1m","remote":{"node":{"attr":""},"initial_connect_timeout":"30s","connect":"true","connections_per_cluster":"3"},"default_keep_alive":"5m"},"security":{"manager":{"filter_bad_defaults":"true"}},"repositories":{"fs":{"compress":"false","chunk_size":"-1b","location":""},"url":{"supported_protocols":["http","https","ftp","file","jar"],"url":"http:"}},"action":{"auto_create_index":"true","search":{"shard_count":{"limit":"9223372036854775807"}},"destructive_requires_name":"false","master":{"force_local":"false"}},"client":{"type":"node","transport":{"ignore_cluster_name":"false","nodes_sampler_interval":"5s","sniff":"false","ping_timeout":"5s"}},"rest":{"action":{"multi":{"allow_explicit_index":"true"}}},"cache":{"recycler":{"page":{"limit":{"heap":"10%"},"type":"CONCURRENT","weight":{"longs":"1.0","ints":"1.0","bytes":"1.0","objects":"0.1"}}}},"resource":{"reload":{"enabled":"true","interval":{"low":"60s","high":"5s","medium":"30s"}}},"thread_pool":{"force_merge":{"queue_size":"-1","size":"1"},"fetch_shard_started":{"core":"1","max":"8","keep_alive":"5m"},"listener":{"queue_size":"-1","size":"2"},"index":{"queue_size":"200","size":"4"},"refresh":{"core":"1","max":"2","keep_alive":"5m"},"generic":{"core":"4","max":"128","keep_alive":"30s"},"warmer":{"core":"1","max":"2","keep_alive":"5m"},"search":{"queue_size":"1000","size":"7"},"fetch_shard_store":{"core":"1","max":"8","keep_alive":"5m"},"flush":{"core":"1","max":"2","keep_alive":"5m"},"management":{"core":"1","max":"5","keep_alive":"5m"},"get":{"queue_size":"1000","size":"4"},"bulk":{"queue_size":"200","size":"4"},"estimated_time_interval":"200ms","snapshot":{"core":"1","max":"2","keep_alive":"5m"}},"index":{"codec":"default","store":{"type":"","fs":{"fs_lock":"native"}}},"monitor":{"jvm":{"gc":{"enabled":"true","overhead":{"warn":"50","debug":"10","info":"25"},"refresh_interval":"1s"},"refresh_interval":"1s"},"process":{"refresh_interval":"1s"},"os":{"refresh_interval":"1s"},"fs":{"refresh_interval":"1s"}},"transport":{"tcp":{"reuse_address":"true","connect_timeout":"30s","compress":"false","port":"9300-9400","blocking_server":"false","blocking_client":"false","keep_alive":"true","receive_buffer_size":"-1b","send_buffer_size":"-1b"},"ping_schedule":"-1","publish_port":"-1","connections_per_node":{"recovery":"2","state":"1","bulk":"3","reg":"6","ping":"1"},"tcp_no_delay":"true","tracer":{"exclude":["internal:discovery/zen/fd*","cluster:monitor/nodes/liveness"]},"type":"","netty":{"max_composite_buffer_components":"-1","worker_count":"8","receive_predictor_size":"512kb","receive_predictor_max":"512kb","receive_predictor_min":"512kb","boss_count":"1","max_cumulation_buffer_capacity":"-1b"},"type.default":"netty4"},"script":{"cache":{"max_size":"100","expire":"0ms"},"painless":{"regex":{"enabled":"false"}},"legacy":{"default_lang":"groovy"},"max_size_in_bytes":"65535","update":"false","max_compilations_per_minute":"15","ingest":"false","search":"false","file":"true","inline":"false","auto_reload_enabled":"true","engine":{"painless":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"true","file":"true","inline":"true","inline.update":"true","stored.search":"true","inline.aggs":"true","file.search":"true","stored":"true","stored.ingest":"true","stored.aggs":"true","stored.update":"true","inline.search":"true"},"expression":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"true","file":"true","inline":"true","inline.update":"true","stored.search":"true","inline.aggs":"true","file.search":"true","stored":"true","stored.ingest":"true","stored.aggs":"true","stored.update":"true","inline.search":"true"},"groovy":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"false","file":"true","inline":"false","inline.update":"false","stored.search":"false","inline.aggs":"false","file.search":"true","stored":"false","stored.ingest":"false","stored.aggs":"false","stored.update":"false","inline.search":"false"},"mustache":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"true","file":"true","inline":"true","inline.update":"true","stored.search":"true","inline.aggs":"true","file.search":"true","stored":"true","stored.ingest":"true","stored.aggs":"true","stored.update":"true","inline.search":"true"}},"stored":"false","aggs":"false"},"node":{"data":"true","enable_lucene_segment_infos_trace":"false","local_storage":"true","max_local_storage_nodes":"1","name":"DK8-2Lc","id":{"seed":"0"},"add_lock_id_to_custom_path":"true","portsfile":"false","ingest":"true","master":"true"},"indices":{"cache":{"cleanup_interval":"1m"},"mapping":{"dynamic_timeout":"30s"},"memory":{"interval":"5s","max_index_buffer_size":"-1b","shard_inactive_time":"5m","index_buffer_size":"10%","min_index_buffer_size":"48mb"},"breaker":{"request":{"limit":"60%","type":"memory","overhead":"1.0"},"total":{"limit":"70%"},"fielddata":{"limit":"60%","type":"memory","overhead":"1.03"},"type":"hierarchy"},"fielddata":{"cache":{"size":"-1b"}},"query":{"bool":{"max_clause_count":"1024"},"query_string":{"analyze_wildcard":"false","allowLeadingWildcard":"true"}},"recovery":{"recovery_activity_timeout":"1800000ms","retry_delay_network":"5s","internal_action_timeout":"15m","retry_delay_state_sync":"500ms","internal_action_long_timeout":"1800000ms","max_bytes_per_sec":"40mb"},"requests":{"cache":{"size":"1%","expire":"0ms"}},"store":{"delete":{"shard":{"timeout":"30s"}},"throttle":{"type":"NONE","max_bytes_per_sec":"0b"}},"analysis":{"hunspell":{"dictionary":{"ignore_case":"false","lazy":"false"}}},"queries":{"cache":{"count":"10000","size":"10%","all_segments":"false"}},"ttl":{"interval":"60s"}},"discovery":{"type":"zen","zen":{"commit_timeout":"30s","no_master_block":"write","join_retry_delay":"100ms","join_retry_attempts":"3","ping":{"unicast":{"concurrent_connects":"10","hosts":{"resolve_timeout":"5s"}}},"master_election":{"ignore_non_master_pings":"false","wait_for_joins_timeout":"30000ms"},"send_leave_request":"true","ping_timeout":"3s","join_timeout":"60000ms","publish_diff":{"enable":"true"},"minimum_master_nodes":"-1","hosts_provider":null,"publish_timeout":"30s","fd":{"connect_on_network_disconnect":"false","ping_interval":"1s","ping_retries":"3","register_connection_listener":"true","ping_timeout":"30s"},"max_pings_from_another_master":"3"},"initial_state_timeout":"30s"},"tribe":{"name":"","on_conflict":"any","blocks":{"write":"false","metadata":"false"}},"http":{"tcp":{"reuse_address":"true","keep_alive":"true","blocking_server":"false","receive_buffer_size":"-1b","send_buffer_size":"-1b"},"bind_host":["0.0.0.0"],"cors":{"max-age":"1728000","allow-origin":"","allow-headers":"X-Requested-With,Content-Type,Content-Length","allow-credentials":"false","allow-methods":"OPTIONS,HEAD,GET,POST,PUT,DELETE","enabled":"false"},"max_chunk_size":"8kb","compression_level":"3","reset_cookies":"false","max_initial_line_length":"4kb","type":"","pipelining":"true","enabled":"true","type.default":"netty4","detailed_errors":{"enabled":"true"},"content_type":{"required":"false"},"port":"9200-9300","host":["0.0.0.0"],"publish_port":"-1","max_header_size":"8kb","pipelining.max_events":"10000","tcp_no_delay":"true","compression":"true","publish_host":["0.0.0.0"],"max_content_length":"100mb","netty":{"receive_predictor_size":"64kb","max_composite_buffer_components":"-1","receive_predictor_max":"64kb","worker_count":"8","receive_predictor_min":"64kb","max_cumulation_buffer_capacity":"-1b"}},"gateway":{"recover_after_master_nodes":"0","expected_nodes":"-1","recover_after_data_nodes":"-1","initial_shards":"quorum","expected_data_nodes":"-1","recover_after_time":"0ms","expected_master_nodes":"-1","recover_after_nodes":"-1"}}}`, "5.4.2-merge": `{"persistent":{},"transient":{"cluster":{"routing":{"allocation":{"enable":"ALL"}}}},"defaults":{"cluster":{"routing":{"rebalance":{"enable":"ALL"},"none":{"node_concurrent_incoming_recoveries":"2","node_initial_primaries_recoveries":"4","same_shard":{"host":"false"},"total_shards_per_node":"-1","type":"balanced","disk":{"threshold_enabled":"true","watermark":{"low":"85%","high":"90%"},"include_relocations":"true","reroute_interval":"60s"},"awareness":{"attributes":""},"balance":{"index":"0.55","threshold":"1.0","shard":"0.45"},"node_concurrent_outgoing_recoveries":"2","allow_rebalance":"indices_all_active","cluster_concurrent_rebalance":"2","node_concurrent_recoveries":"2","snapshot":{"relocation_enabled":"false"}}},"indices":{"close":{"enable":"true"}},"nodes":{"reconnect_interval":"10s"},"blocks":{"read_only":"false"},"service":{"slow_task_logging_threshold":"30s"},"name":"elasticsearch","info":{"update":{"interval":"30s","timeout":"15s"}}},"logger":{"level":"INFO"},"bootstrap":{"ctrlhandler":"true","memory_lock":"false","system_call_filter":"true","seccomp":"true"},"processors":"4","network":{"host":["_local_"],"tcp":{"reuse_address":"true","connect_timeout":"30s","blocking":"false","blocking_server":"false","no_delay":"true","blocking_client":"false","keep_alive":"true","receive_buffer_size":"-1b","send_buffer_size":"-1b"},"bind_host":["_local_"],"server":"true","breaker":{"inflight_requests":{"limit":"100%","overhead":"1.0"}},"publish_host":["_local_"]},"pidfile":"","path":{"conf":"","scripts":"","logs":"/usr/share/elasticsearch/logs","shared_data":"","home":"/usr/share/elasticsearch"},"default":{"path":{"logs":"","conf":""}},"search":{"default_search_timeout":"-1","highlight":{"term_vector_multi_value":"true"},"low_level_cancellation":"false","keep_alive_interval":"1m","remote":{"node":{"attr":""},"initial_connect_timeout":"30s","connect":"true","connections_per_cluster":"3"},"default_keep_alive":"5m"},"security":{"manager":{"filter_bad_defaults":"true"}},"repositories":{"fs":{"compress":"false","chunk_size":"-1b","location":""},"url":{"supported_protocols":["http","https","ftp","file","jar"],"url":"http:"}},"action":{"auto_create_index":"true","search":{"shard_count":{"limit":"9223372036854775807"}},"destructive_requires_name":"false","master":{"force_local":"false"}},"client":{"type":"node","transport":{"ignore_cluster_name":"false","nodes_sampler_interval":"5s","sniff":"false","ping_timeout":"5s"}},"rest":{"action":{"multi":{"allow_explicit_index":"true"}}},"cache":{"recycler":{"page":{"limit":{"heap":"10%"},"type":"CONCURRENT","weight":{"longs":"1.0","ints":"1.0","bytes":"1.0","objects":"0.1"}}}},"resource":{"reload":{"enabled":"true","interval":{"low":"60s","high":"5s","medium":"30s"}}},"thread_pool":{"force_merge":{"queue_size":"-1","size":"1"},"fetch_shard_started":{"core":"1","max":"8","keep_alive":"5m"},"listener":{"queue_size":"-1","size":"2"},"index":{"queue_size":"200","size":"4"},"refresh":{"core":"1","max":"2","keep_alive":"5m"},"generic":{"core":"4","max":"128","keep_alive":"30s"},"warmer":{"core":"1","max":"2","keep_alive":"5m"},"search":{"queue_size":"1000","size":"7"},"fetch_shard_store":{"core":"1","max":"8","keep_alive":"5m"},"flush":{"core":"1","max":"2","keep_alive":"5m"},"management":{"core":"1","max":"5","keep_alive":"5m"},"get":{"queue_size":"1000","size":"4"},"bulk":{"queue_size":"200","size":"4"},"estimated_time_interval":"200ms","snapshot":{"core":"1","max":"2","keep_alive":"5m"}},"index":{"codec":"default","store":{"type":"","fs":{"fs_lock":"native"}}},"monitor":{"jvm":{"gc":{"enabled":"true","overhead":{"warn":"50","debug":"10","info":"25"},"refresh_interval":"1s"},"refresh_interval":"1s"},"process":{"refresh_interval":"1s"},"os":{"refresh_interval":"1s"},"fs":{"refresh_interval":"1s"}},"transport":{"tcp":{"reuse_address":"true","connect_timeout":"30s","compress":"false","port":"9300-9400","blocking_server":"false","blocking_client":"false","keep_alive":"true","receive_buffer_size":"-1b","send_buffer_size":"-1b"},"ping_schedule":"-1","publish_port":"-1","connections_per_node":{"recovery":"2","state":"1","bulk":"3","reg":"6","ping":"1"},"tcp_no_delay":"true","tracer":{"exclude":["internal:discovery/zen/fd*","cluster:monitor/nodes/liveness"]},"type":"","netty":{"max_composite_buffer_components":"-1","worker_count":"8","receive_predictor_size":"512kb","receive_predictor_max":"512kb","receive_predictor_min":"512kb","boss_count":"1","max_cumulation_buffer_capacity":"-1b"},"type.default":"netty4"},"script":{"cache":{"max_size":"100","expire":"0ms"},"painless":{"regex":{"enabled":"false"}},"legacy":{"default_lang":"groovy"},"max_size_in_bytes":"65535","update":"false","max_compilations_per_minute":"15","ingest":"false","search":"false","file":"true","inline":"false","auto_reload_enabled":"true","engine":{"painless":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"true","file":"true","inline":"true","inline.update":"true","stored.search":"true","inline.aggs":"true","file.search":"true","stored":"true","stored.ingest":"true","stored.aggs":"true","stored.update":"true","inline.search":"true"},"expression":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"true","file":"true","inline":"true","inline.update":"true","stored.search":"true","inline.aggs":"true","file.search":"true","stored":"true","stored.ingest":"true","stored.aggs":"true","stored.update":"true","inline.search":"true"},"groovy":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"false","file":"true","inline":"false","inline.update":"false","stored.search":"false","inline.aggs":"false","file.search":"true","stored":"false","stored.ingest":"false","stored.aggs":"false","stored.update":"false","inline.search":"false"},"mustache":{"file.ingest":"true","file.update":"true","file.aggs":"true","inline.ingest":"true","file":"true","inline":"true","inline.update":"true","stored.search":"true","inline.aggs":"true","file.search":"true","stored":"true","stored.ingest":"true","stored.aggs":"true","stored.update":"true","inline.search":"true"}},"stored":"false","aggs":"false"},"node":{"data":"true","enable_lucene_segment_infos_trace":"false","local_storage":"true","max_local_storage_nodes":"1","name":"DK8-2Lc","id":{"seed":"0"},"add_lock_id_to_custom_path":"true","portsfile":"false","ingest":"true","master":"true"},"indices":{"cache":{"cleanup_interval":"1m"},"mapping":{"dynamic_timeout":"30s"},"memory":{"interval":"5s","max_index_buffer_size":"-1b","shard_inactive_time":"5m","index_buffer_size":"10%","min_index_buffer_size":"48mb"},"breaker":{"request":{"limit":"60%","type":"memory","overhead":"1.0"},"total":{"limit":"70%"},"fielddata":{"limit":"60%","type":"memory","overhead":"1.03"},"type":"hierarchy"},"fielddata":{"cache":{"size":"-1b"}},"query":{"bool":{"max_clause_count":"1024"},"query_string":{"analyze_wildcard":"false","allowLeadingWildcard":"true"}},"recovery":{"recovery_activity_timeout":"1800000ms","retry_delay_network":"5s","internal_action_timeout":"15m","retry_delay_state_sync":"500ms","internal_action_long_timeout":"1800000ms","max_bytes_per_sec":"40mb"},"requests":{"cache":{"size":"1%","expire":"0ms"}},"store":{"delete":{"shard":{"timeout":"30s"}},"throttle":{"type":"NONE","max_bytes_per_sec":"0b"}},"analysis":{"hunspell":{"dictionary":{"ignore_case":"false","lazy":"false"}}},"queries":{"cache":{"count":"10000","size":"10%","all_segments":"false"}},"ttl":{"interval":"60s"}},"discovery":{"type":"zen","zen":{"commit_timeout":"30s","no_master_block":"write","join_retry_delay":"100ms","join_retry_attempts":"3","ping":{"unicast":{"concurrent_connects":"10","hosts":{"resolve_timeout":"5s"}}},"master_election":{"ignore_non_master_pings":"false","wait_for_joins_timeout":"30000ms"},"send_leave_request":"true","ping_timeout":"3s","join_timeout":"60000ms","publish_diff":{"enable":"true"},"minimum_master_nodes":"-1","hosts_provider":null,"publish_timeout":"30s","fd":{"connect_on_network_disconnect":"false","ping_interval":"1s","ping_retries":"3","register_connection_listener":"true","ping_timeout":"30s"},"max_pings_from_another_master":"3"},"initial_state_timeout":"30s"},"tribe":{"name":"","on_conflict":"any","blocks":{"write":"false","metadata":"false"}},"http":{"tcp":{"reuse_address":"true","keep_alive":"true","blocking_server":"false","receive_buffer_size":"-1b","send_buffer_size":"-1b"},"bind_host":["0.0.0.0"],"cors":{"max-age":"1728000","allow-origin":"","allow-headers":"X-Requested-With,Content-Type,Content-Length","allow-credentials":"false","allow-methods":"OPTIONS,HEAD,GET,POST,PUT,DELETE","enabled":"false"},"max_chunk_size":"8kb","compression_level":"3","reset_cookies":"false","max_initial_line_length":"4kb","type":"","pipelining":"true","enabled":"true","type.default":"netty4","detailed_errors":{"enabled":"true"},"content_type":{"required":"false"},"port":"9200-9300","host":["0.0.0.0"],"publish_port":"-1","max_header_size":"8kb","pipelining.max_events":"10000","tcp_no_delay":"true","compression":"true","publish_host":["0.0.0.0"],"max_content_length":"100mb","netty":{"receive_predictor_size":"64kb","max_composite_buffer_components":"-1","receive_predictor_max":"64kb","worker_count":"8","receive_predictor_min":"64kb","max_cumulation_buffer_capacity":"-1b"}},"gateway":{"recover_after_master_nodes":"0","expected_nodes":"-1","recover_after_data_nodes":"-1","initial_shards":"quorum","expected_data_nodes":"-1","recover_after_time":"0ms","expected_master_nodes":"-1","recover_after_nodes":"-1"}}}`} for ver, out := range tcs { for hn, handler := range map[string]http.Handler{ "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, out) }), } { ts := httptest.NewServer(handler) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("Failed to parse URL: %s", err) } c := NewClusterSettings(log.NewNopLogger(), http.DefaultClient, u) nsr, err := c.fetchAndDecodeClusterSettingsStats() if err != nil { t.Fatalf("Failed to fetch or decode cluster settings stats: %s", err) } t.Logf("[%s/%s] Cluster Settings Stats Response: %+v", hn, ver, nsr) if nsr.Cluster.Routing.Allocation.Enabled != "ALL" { t.Errorf("Wrong setting for cluster routing allocation enabled") } } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/indices.go000066400000000000000000001053731362377146000247410ustar00rootroot00000000000000package collector import ( "encoding/json" "fmt" "net/http" "net/url" "path" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/justwatchcom/elasticsearch_exporter/pkg/clusterinfo" "github.com/prometheus/client_golang/prometheus" ) type labels struct { keys func(...string) []string values func(*clusterinfo.Response, ...string) []string } type indexMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(indexStats IndexStatsIndexResponse) float64 Labels labels } type shardMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(data IndexStatsIndexShardsDetailResponse) float64 Labels labels } // Indices information struct type Indices struct { logger log.Logger client *http.Client url *url.URL shards bool clusterInfoCh chan *clusterinfo.Response lastClusterInfo *clusterinfo.Response up prometheus.Gauge totalScrapes prometheus.Counter jsonParseFailures prometheus.Counter indexMetrics []*indexMetric shardMetrics []*shardMetric } // NewIndices defines Indices Prometheus metrics func NewIndices(logger log.Logger, client *http.Client, url *url.URL, shards bool) *Indices { indexLabels := labels{ keys: func(...string) []string { return []string{"index", "cluster"} }, values: func(lastClusterinfo *clusterinfo.Response, s ...string) []string { if lastClusterinfo != nil { return append(s, lastClusterinfo.ClusterName) } // this shouldn't happen, as the clusterinfo Retriever has a blocking // Run method. It blocks until the first clusterinfo call has succeeded return append(s, "unknown_cluster") }, } shardLabels := labels{ keys: func(...string) []string { return []string{"index", "shard", "node", "cluster"} }, values: func(lastClusterinfo *clusterinfo.Response, s ...string) []string { if lastClusterinfo != nil { return append(s, lastClusterinfo.ClusterName) } // this shouldn't happen, as the clusterinfo Retriever has a blocking // Run method. It blocks until the first clusterinfo call has succeeded return append(s, "unknown_cluster") }, } indices := &Indices{ logger: logger, client: client, url: url, shards: shards, clusterInfoCh: make(chan *clusterinfo.Response), lastClusterInfo: &clusterinfo.Response{ ClusterName: "unknown_cluster", }, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "index_stats", "up"), Help: "Was the last scrape of the ElasticSearch index endpoint successful.", }), totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "index_stats", "total_scrapes"), Help: "Current total ElasticSearch index scrapes.", }), jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "index_stats", "json_parse_failures"), Help: "Number of errors while parsing JSON.", }), indexMetrics: []*indexMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "docs_primary"), "Count of documents with only primary shards", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Docs.Count) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "deleted_docs_primary"), "Count of deleted documents with only primary shards", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Docs.Deleted) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "docs_total"), "Total count of documents", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Docs.Count) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "deleted_docs_total"), "Total count of deleted documents", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Docs.Deleted) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "store_size_bytes_primary"), "Current total size of stored index data in bytes with only primary shards on all nodes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Store.SizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "store_size_bytes_total"), "Current total size of stored index data in bytes with all shards on all nodes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Store.SizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_count_primary"), "Current number of segments with only primary shards on all nodes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.Count) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_count_total"), "Current number of segments with all shards on all nodes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.Count) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_memory_bytes_primary"), "Current size of segments with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.MemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_memory_bytes_total"), "Current size of segments with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.MemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_terms_memory_primary"), "Current size of terms with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.TermsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_terms_memory_total"), "Current number of terms with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.TermsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_fields_memory_bytes_primary"), "Current size of fields with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.StoredFieldsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_fields_memory_bytes_total"), "Current size of fields with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.StoredFieldsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_term_vectors_memory_primary_bytes"), "Current size of term vectors with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.TermVectorsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_term_vectors_memory_total_bytes"), "Current size of term vectors with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.TermVectorsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_norms_memory_bytes_primary"), "Current size of norms with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.NormsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_norms_memory_bytes_total"), "Current size of norms with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.NormsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_points_memory_bytes_primary"), "Current size of points with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.PointsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_points_memory_bytes_total"), "Current size of points with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.PointsMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_doc_values_memory_bytes_primary"), "Current size of doc values with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.DocValuesMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_doc_values_memory_bytes_total"), "Current size of doc values with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.DocValuesMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_index_writer_memory_bytes_primary"), "Current size of index writer with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.IndexWriterMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_index_writer_memory_bytes_total"), "Current size of index writer with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.IndexWriterMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_version_map_memory_bytes_primary"), "Current size of version map with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.VersionMapMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_version_map_memory_bytes_total"), "Current size of version map with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.VersionMapMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_fixed_bit_set_memory_bytes_primary"), "Current size of fixed bit with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Segments.FixedBitSetMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segment_fixed_bit_set_memory_bytes_total"), "Current size of fixed bit with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Segments.FixedBitSetMemoryInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "completion_bytes_primary"), "Current size of completion with only primary shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Primaries.Completion.SizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "completion_bytes_total"), "Current size of completion with all shards on all nodes in bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Completion.SizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_query_time_seconds_total"), "Total search query time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.QueryTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_query_total"), "Total number of queries", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.QueryTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_fetch_time_seconds_total"), "Total search fetch time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.FetchTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_fetch_total"), "Total search fetch count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.FetchTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_scroll_time_seconds_total"), "Total search scroll time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.ScrollTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_scroll_current"), "Current search scroll count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.ScrollCurrent) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_scroll_total"), "Total search scroll count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.ScrollTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_suggest_time_seconds_total"), "Total search suggest time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.SuggestTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "search_suggest_total"), "Total search suggest count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Search.SuggestTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "indexing_index_time_seconds_total"), "Total indexing index time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Indexing.IndexTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "indexing_index_total"), "Total indexing index count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Indexing.IndexTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "indexing_delete_time_seconds_total"), "Total indexing delete time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Indexing.DeleteTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "indexing_delete_total"), "Total indexing delete count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Indexing.DeleteTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "indexing_noop_update_total"), "Total indexing no-op update count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Indexing.NoopUpdateTotal) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "indexing_throttle_time_seconds_total"), "Total indexing throttle time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Indexing.ThrottleTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "get_time_seconds_total"), "Total get time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Get.TimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "get_total"), "Total get count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Get.Total) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "merge_time_seconds_total"), "Total merge time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Merges.TotalTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "merge_total"), "Total merge count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Merges.Total) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "merge_throttle_time_seconds_total"), "Total merge I/O throttle time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Merges.TotalThrottledTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "merge_stopped_time_seconds_total"), "Total large merge stopped time in seconds, allowing smaller merges to complete", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Merges.TotalStoppedTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "merge_auto_throttle_bytes_total"), "Total bytes that were auto-throttled during merging", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Merges.TotalAutoThrottleInBytes) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "refresh_time_seconds_total"), "Total refresh time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Refresh.TotalTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "refresh_total"), "Total refresh count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Refresh.Total) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "flush_time_seconds_total"), "Total flush time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Flush.TotalTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "flush_total"), "Total flush count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Flush.Total) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "warmer_time_seconds_total"), "Total warmer time in seconds", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Warmer.TotalTimeInMillis) / 1000 }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "warmer_total"), "Total warmer count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Warmer.Total) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "query_cache_memory_bytes_total"), "Total query cache memory bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.QueryCache.MemorySizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "query_cache_size"), "Total query cache size", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.QueryCache.CacheSize) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "query_cache_hits_total"), "Total query cache hits count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.QueryCache.HitCount) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "query_cache_misses_total"), "Total query cache misses count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.QueryCache.MissCount) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "query_cache_caches_total"), "Total query cache caches count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.QueryCache.CacheCount) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "query_cache_evictions_total"), "Total query cache evictions count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.QueryCache.Evictions) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "request_cache_memory_bytes_total"), "Total request cache memory bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.RequestCache.MemorySizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "request_cache_hits_total"), "Total request cache hits count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.RequestCache.HitCount) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "request_cache_misses_total"), "Total request cache misses count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.RequestCache.MissCount) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "request_cache_evictions_total"), "Total request cache evictions count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.RequestCache.Evictions) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "fielddata_memory_bytes_total"), "Total fielddata memory bytes", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Fielddata.MemorySizeInBytes) }, Labels: indexLabels, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "fielddata_evictions_total"), "Total fielddata evictions count", indexLabels.keys(), nil, ), Value: func(indexStats IndexStatsIndexResponse) float64 { return float64(indexStats.Total.Fielddata.Evictions) }, Labels: indexLabels, }, }, shardMetrics: []*shardMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "shared_docs"), "Count of documents on this shard", shardLabels.keys(), nil, ), Value: func(data IndexStatsIndexShardsDetailResponse) float64 { return float64(data.Docs.Count) }, Labels: shardLabels, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "shards_docs_deleted"), "Count of deleted documents on this shard", shardLabels.keys(), nil, ), Value: func(data IndexStatsIndexShardsDetailResponse) float64 { return float64(data.Docs.Deleted) }, Labels: shardLabels, }, }, } // start go routine to fetch clusterinfo updates and save them to lastClusterinfo go func() { _ = level.Debug(logger).Log("msg", "starting cluster info receive loop") for ci := range indices.clusterInfoCh { _ = level.Debug(logger).Log("msg", "received cluster info update", "cluster", ci.ClusterName) if ci != nil { indices.lastClusterInfo = ci } } _ = level.Debug(logger).Log("msg", "exiting cluster info receive loop") }() return indices } // ClusterLabelUpdates returns a pointer to a channel to receive cluster info updates. It implements the // (not exported) clusterinfo.consumer interface func (i *Indices) ClusterLabelUpdates() *chan *clusterinfo.Response { return &i.clusterInfoCh } // String implements the stringer interface. It is part of the clusterinfo.consumer interface func (i *Indices) String() string { return namespace + "indices" } // Describe add Indices metrics descriptions func (i *Indices) Describe(ch chan<- *prometheus.Desc) { for _, metric := range i.indexMetrics { ch <- metric.Desc } ch <- i.up.Desc() ch <- i.totalScrapes.Desc() ch <- i.jsonParseFailures.Desc() } func (i *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse u := *i.url u.Path = path.Join(u.Path, "/_all/_stats") if i.shards { u.RawQuery = "level=shards" } res, err := i.client.Get(u.String()) if err != nil { return isr, fmt.Errorf("failed to get index stats from %s://%s:%s%s: %s", u.Scheme, u.Hostname(), u.Port(), u.Path, err) } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(i.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { i.jsonParseFailures.Inc() return isr, err } return isr, nil } // Collect gets Indices metric values func (i *Indices) Collect(ch chan<- prometheus.Metric) { i.totalScrapes.Inc() defer func() { ch <- i.up ch <- i.totalScrapes ch <- i.jsonParseFailures }() // indices indexStatsResp, err := i.fetchAndDecodeIndexStats() if err != nil { i.up.Set(0) _ = level.Warn(i.logger).Log( "msg", "failed to fetch and decode index stats", "err", err, ) return } i.totalScrapes.Inc() i.up.Set(1) // Index stats for indexName, indexStats := range indexStatsResp.Indices { for _, metric := range i.indexMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(indexStats), metric.Labels.values(i.lastClusterInfo, indexName)..., ) } if i.shards { for _, metric := range i.shardMetrics { // gaugeVec := prometheus.NewGaugeVec(metric.Opts, metric.Labels) for shardNumber, shards := range indexStats.Shards { for _, shard := range shards { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(shard), metric.Labels.values(i.lastClusterInfo, indexName, shardNumber, shard.Routing.Node)..., ) } } } } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/indices_response.go000066400000000000000000000217471362377146000266610ustar00rootroot00000000000000package collector // indexStatsResponse is a representation of a Elasticsearch Index Stats type indexStatsResponse struct { Shards IndexStatsShardsResponse `json:"_shards"` All IndexStatsIndexResponse `json:"_all"` Indices map[string]IndexStatsIndexResponse `json:"indices"` } // IndexStatsShardsResponse defines index stats shards information structure type IndexStatsShardsResponse struct { Total int64 `json:"total"` Successful int64 `json:"successful"` Failed int64 `json:"failed"` } // IndexStatsIndexResponse defines index stats index information structure type IndexStatsIndexResponse struct { Primaries IndexStatsIndexDetailResponse `json:"primaries"` Total IndexStatsIndexDetailResponse `json:"total"` Shards map[string][]IndexStatsIndexShardsDetailResponse `json:"shards"` } // IndexStatsIndexDetailResponse defines index stats index details information structure type IndexStatsIndexDetailResponse struct { Docs IndexStatsIndexDocsResponse `json:"docs"` Store IndexStatsIndexStoreResponse `json:"store"` Indexing IndexStatsIndexIndexingResponse `json:"indexing"` Get IndexStatsIndexGetResponse `json:"get"` Search IndexStatsIndexSearchResponse `json:"search"` Merges IndexStatsIndexMergesResponse `json:"merges"` Refresh IndexStatsIndexRefreshResponse `json:"refresh"` Flush IndexStatsIndexFlushResponse `json:"flush"` Warmer IndexStatsIndexWarmerResponse `json:"warmer"` QueryCache IndexStatsIndexQueryCacheResponse `json:"query_cache"` Fielddata IndexStatsIndexFielddataResponse `json:"fielddata"` Completion IndexStatsIndexCompletionResponse `json:"completion"` Segments IndexStatsIndexSegmentsResponse `json:"segments"` Translog IndexStatsIndexTranslogResponse `json:"translog"` RequestCache IndexStatsIndexRequestCacheResponse `json:"request_cache"` Recovery IndexStatsIndexRecoveryResponse `json:"recovery"` } // IndexStatsIndexShardsDetailResponse defines index stats index shard details information structure type IndexStatsIndexShardsDetailResponse struct { *IndexStatsIndexDetailResponse Routing IndexStatsIndexRoutingResponse `json:"routing"` } // IndexStatsIndexRoutingResponse defines index stats index routing information structure type IndexStatsIndexRoutingResponse struct { Node string `json:"node"` Primary bool `json:"primary"` } // IndexStatsIndexDocsResponse defines index stats index documents information structure type IndexStatsIndexDocsResponse struct { Count int64 `json:"count"` Deleted int64 `json:"deleted"` } // IndexStatsIndexStoreResponse defines index stats index store information structure type IndexStatsIndexStoreResponse struct { SizeInBytes int64 `json:"size_in_bytes"` ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` } // IndexStatsIndexIndexingResponse defines index stats index indexing information structure type IndexStatsIndexIndexingResponse struct { IndexTotal int64 `json:"index_total"` IndexTimeInMillis int64 `json:"index_time_in_millis"` IndexCurrent int64 `json:"index_current"` IndexFailed int64 `json:"index_failed"` DeleteTotal int64 `json:"delete_total"` DeleteTimeInMillis int64 `json:"delete_time_in_millis"` DeleteCurrent int64 `json:"delete_current"` NoopUpdateTotal int64 `json:"noop_update_total"` IsThrottled bool `json:"is_throttled"` ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` } // IndexStatsIndexGetResponse defines index stats index get information structure type IndexStatsIndexGetResponse struct { Total int64 `json:"total"` TimeInMillis int64 `json:"time_in_millis"` ExistsTotal int64 `json:"exists_total"` ExistsTimeInMillis int64 `json:"exists_time_in_millis"` MissingTotal int64 `json:"missing_total"` MissingTimeInMillis int64 `json:"missing_time_in_millis"` Current int64 `json:"current"` } // IndexStatsIndexSearchResponse defines index stats index search information structure type IndexStatsIndexSearchResponse struct { OpenContexts int64 `json:"open_contexts"` QueryTotal int64 `json:"query_total"` QueryTimeInMillis int64 `json:"query_time_in_millis"` QueryCurrent int64 `json:"query_current"` FetchTotal int64 `json:"fetch_total"` FetchTimeInMillis int64 `json:"fetch_time_in_millis"` FetchCurrent int64 `json:"fetch_current"` ScrollTotal int64 `json:"scroll_total"` ScrollTimeInMillis int64 `json:"scroll_time_in_millis"` ScrollCurrent int64 `json:"scroll_current"` SuggestTotal int64 `json:"suggest_total"` SuggestTimeInMillis int64 `json:"suggest_time_in_millis"` SuggestCurrent int64 `json:"suggest_current"` } // IndexStatsIndexMergesResponse defines index stats index merges information structure type IndexStatsIndexMergesResponse struct { Current int64 `json:"current"` CurrentDocs int64 `json:"current_docs"` CurrentSizeInBytes int64 `json:"current_size_in_bytes"` Total int64 `json:"total"` TotalTimeInMillis int64 `json:"total_time_in_millis"` TotalDocs int64 `json:"total_docs"` TotalSizeInBytes int64 `json:"total_size_in_bytes"` TotalStoppedTimeInMillis int64 `json:"total_stopped_time_in_millis"` TotalThrottledTimeInMillis int64 `json:"total_throttled_time_in_millis"` TotalAutoThrottleInBytes int64 `json:"total_auto_throttle_in_bytes"` } // IndexStatsIndexRefreshResponse defines index stats index refresh information structure type IndexStatsIndexRefreshResponse struct { Total int64 `json:"total"` TotalTimeInMillis int64 `json:"total_time_in_millis"` Listeners int64 `json:"listeners"` } // IndexStatsIndexFlushResponse defines index stats index flush information structure type IndexStatsIndexFlushResponse struct { Total int64 `json:"total"` TotalTimeInMillis int64 `json:"total_time_in_millis"` } // IndexStatsIndexWarmerResponse defines index stats index warmer information structure type IndexStatsIndexWarmerResponse struct { Current int64 `json:"current"` Total int64 `json:"total"` TotalTimeInMillis int64 `json:"total_time_in_millis"` } // IndexStatsIndexQueryCacheResponse defines index stats index query cache information structure type IndexStatsIndexQueryCacheResponse struct { MemorySizeInBytes int64 `json:"memory_size_in_bytes"` TotalCount int64 `json:"total_count"` HitCount int64 `json:"hit_count"` MissCount int64 `json:"miss_count"` CacheSize int64 `json:"cache_size"` CacheCount int64 `json:"cache_count"` Evictions int64 `json:"evictions"` } // IndexStatsIndexFielddataResponse defines index stats index fielddata information structure type IndexStatsIndexFielddataResponse struct { MemorySizeInBytes int64 `json:"memory_size_in_bytes"` Evictions int64 `json:"evictions"` } // IndexStatsIndexCompletionResponse defines index stats index completion information structure type IndexStatsIndexCompletionResponse struct { SizeInBytes int64 `json:"size_in_bytes"` } // IndexStatsIndexSegmentsResponse defines index stats index segments information structure type IndexStatsIndexSegmentsResponse struct { Count int64 `json:"count"` MemoryInBytes int64 `json:"memory_in_bytes"` TermsMemoryInBytes int64 `json:"terms_memory_in_bytes"` StoredFieldsMemoryInBytes int64 `json:"stored_fields_memory_in_bytes"` TermVectorsMemoryInBytes int64 `json:"term_vectors_memory_in_bytes"` NormsMemoryInBytes int64 `json:"norms_memory_in_bytes"` PointsMemoryInBytes int64 `json:"points_memory_in_bytes"` DocValuesMemoryInBytes int64 `json:"doc_values_memory_in_bytes"` IndexWriterMemoryInBytes int64 `json:"index_writer_memory_in_bytes"` VersionMapMemoryInBytes int64 `json:"version_map_memory_in_bytes"` FixedBitSetMemoryInBytes int64 `json:"fixed_bit_set_memory_in_bytes"` MaxUnsafeAutoIDTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` } // IndexStatsIndexTranslogResponse defines index stats index translog information structure type IndexStatsIndexTranslogResponse struct { Operations int64 `json:"operations"` SizeInBytes int64 `json:"size_in_bytes"` } // IndexStatsIndexRequestCacheResponse defines index stats index request cache information structure type IndexStatsIndexRequestCacheResponse struct { MemorySizeInBytes int64 `json:"memory_size_in_bytes"` Evictions int64 `json:"evictions"` HitCount int64 `json:"hit_count"` MissCount int64 `json:"miss_count"` } // IndexStatsIndexRecoveryResponse defines index stats index recovery information structure type IndexStatsIndexRecoveryResponse struct { CurrentAsSource int64 `json:"current_as_source"` CurrentAsTarget int64 `json:"current_as_target"` ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` } prometheus-elasticsearch-exporter-1.1.0+ds/collector/indices_settings.go000066400000000000000000000065531362377146000266610ustar00rootroot00000000000000package collector import ( "encoding/json" "fmt" "net/http" "net/url" "path" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) // IndicesSettings information struct type IndicesSettings struct { logger log.Logger client *http.Client url *url.URL up prometheus.Gauge readOnlyIndices prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter } // NewIndicesSettings defines Indices Settings Prometheus metrics func NewIndicesSettings(logger log.Logger, client *http.Client, url *url.URL) *IndicesSettings { return &IndicesSettings{ logger: logger, client: client, url: url, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "up"), Help: "Was the last scrape of the ElasticSearch Indices Settings endpoint successful.", }), totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "total_scrapes"), Help: "Current total ElasticSearch Indices Settings scrapes.", }), readOnlyIndices: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "read_only_indices"), Help: "Current number of read only indices within cluster", }), jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "indices_settings_stats", "json_parse_failures"), Help: "Number of errors while parsing JSON.", }), } } // Describe add Snapshots metrics descriptions func (cs *IndicesSettings) Describe(ch chan<- *prometheus.Desc) { ch <- cs.up.Desc() ch <- cs.totalScrapes.Desc() ch <- cs.readOnlyIndices.Desc() ch <- cs.jsonParseFailures.Desc() } func (cs *IndicesSettings) getAndParseURL(u *url.URL, data interface{}) error { res, err := cs.client.Get(u.String()) if err != nil { return fmt.Errorf("failed to get from %s://%s:%s%s: %s", u.Scheme, u.Hostname(), u.Port(), u.Path, err) } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(cs.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(data); err != nil { cs.jsonParseFailures.Inc() return err } return nil } func (cs *IndicesSettings) fetchAndDecodeIndicesSettings() (IndicesSettingsResponse, error) { u := *cs.url u.Path = path.Join(u.Path, "/_all/_settings") var asr IndicesSettingsResponse err := cs.getAndParseURL(&u, &asr) if err != nil { return asr, err } return asr, err } // Collect gets all indices settings metric values func (cs *IndicesSettings) Collect(ch chan<- prometheus.Metric) { cs.totalScrapes.Inc() defer func() { ch <- cs.up ch <- cs.totalScrapes ch <- cs.jsonParseFailures ch <- cs.readOnlyIndices }() asr, err := cs.fetchAndDecodeIndicesSettings() if err != nil { cs.readOnlyIndices.Set(0) cs.up.Set(0) _ = level.Warn(cs.logger).Log( "msg", "failed to fetch and decode cluster settings stats", "err", err, ) return } cs.up.Set(1) var c int for _, value := range asr { if value.Settings.IndexInfo.Blocks.ReadOnly == "true" { c++ } } cs.readOnlyIndices.Set(float64(c)) } prometheus-elasticsearch-exporter-1.1.0+ds/collector/indices_settings_response.go000066400000000000000000000012101362377146000305600ustar00rootroot00000000000000package collector // IndicesSettingsResponse is a representation of Elasticsearch Settings for each Index type IndicesSettingsResponse map[string]Index // Index defines the struct of the tree for the settings of each index type Index struct { Settings Settings `json:"settings"` } // Settings defines current index settings type Settings struct { IndexInfo IndexInfo `json:"index"` } // IndexInfo defines the blocks of the current index type IndexInfo struct { Blocks Blocks `json:"blocks"` } // Blocks defines whether current index has read_only_allow_delete enabled type Blocks struct { ReadOnly string `json:"read_only_allow_delete"` } prometheus-elasticsearch-exporter-1.1.0+ds/collector/indices_settings_test.go000066400000000000000000000056421362377146000277160ustar00rootroot00000000000000package collector import ( "fmt" "net/http" "net/http/httptest" "net/url" "testing" "github.com/go-kit/kit/log" ) func TestIndicesSettings(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION // curl -XPUT http://localhost:9200/twitter // curl -XPUT http://localhost:9200/facebook // curl -XPUT http://localhost:9200/instagram // curl -XPUT http://localhost:9200/viber // curl -XPUT http://localhost:9200/instagram/_settings --header "Content-Type: application/json" -d ' // { // "index": { // "blocks": { // "read_only_allow_delete": "true" // } // } // }' // curl -XPUT http://localhost:9200/twitter/_settings --header "Content-Type: application/json" -d ' // { // "index": { // "blocks": { // "read_only_allow_delete": "true" // } // } // }' // curl http://localhost:9200/_all/_settings tcs := map[string]string{ "6.5.4": `{"viber":{"settings":{"index":{"creation_date":"1548066996192","number_of_shards":"5","number_of_replicas":"1","uuid":"kt2cGV-yQRaloESpqj2zsg","version":{"created":"6050499"},"provided_name":"viber"}}},"facebook":{"settings":{"index":{"creation_date":"1548066984670","number_of_shards":"5","number_of_replicas":"1","uuid":"jrU8OWQZQD--9v5eg0tjbg","version":{"created":"6050499"},"provided_name":"facebook"}}},"twitter":{"settings":{"index":{"number_of_shards":"5","blocks":{"read_only_allow_delete":"true"},"provided_name":"twitter","creation_date":"1548066697559","number_of_replicas":"1","uuid":"-sqtc4fVRrS2jHJCZ2hQ9Q","version":{"created":"6050499"}}}},"instagram":{"settings":{"index":{"number_of_shards":"5","blocks":{"read_only_allow_delete":"true"},"provided_name":"instagram","creation_date":"1548066991932","number_of_replicas":"1","uuid":"WeGWaxa_S3KrgE5SZHolTw","version":{"created":"6050499"}}}}}`, } for ver, out := range tcs { for hn, handler := range map[string]http.Handler{ "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, out) }), } { ts := httptest.NewServer(handler) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("Failed to parse URL: %s", err) } c := NewIndicesSettings(log.NewNopLogger(), http.DefaultClient, u) nsr, err := c.fetchAndDecodeIndicesSettings() if err != nil { t.Fatalf("Failed to fetch or decode indices settings: %s", err) } t.Logf("[%s/%s] All Indices Settings Response: %+v", hn, ver, nsr) // if nsr.Cluster.Routing.Allocation.Enabled != "ALL" { // t.Errorf("Wrong setting for cluster routing allocation enabled") // } var counter int for key, value := range nsr { if value.Settings.IndexInfo.Blocks.ReadOnly == "true" { counter++ if key != "instagram" && key != "twitter" { t.Errorf("Wrong read_only index") } } } if counter != 2 { t.Errorf("Wrong number of read_only indexes") } } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/indices_test.go000066400000000000000000002553401362377146000260000ustar00rootroot00000000000000package collector import ( "fmt" "net/http" "net/http/httptest" "net/url" "testing" "github.com/go-kit/kit/log" ) func TestIndices(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine // curl -XPUT http://localhost:9200/foo_1/type1/1 -d '{"title":"abc","content":"hello"}' // curl -XPUT http://localhost:9200/foo_1/type1/2 -d '{"title":"def","content":"world"}' // curl -XPUT http://localhost:9200/foo_2/type1/1 -d '{"title":"abc001","content":"hello001"}' // curl -XPUT http://localhost:9200/foo_2/type1/2 -d '{"title":"def002","content":"world002"}' // curl -XPUT http://localhost:9200/foo_2/type1/3 -d '{"title":"def003","content":"world003"}' // curl http://localhost:9200/_all/_stats ti := map[string]string{ "1.7.6": `{"_shards":{"total":20,"successful":10,"failed":0},"_all":{"primaries":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":13798,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":52,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":5,"total_time_in_millis":163},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":18410,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":671088640,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":13798,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":52,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":5,"total_time_in_millis":163},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":18410,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":671088640,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":8207,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":38},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":8207,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":38},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":5591,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":2,"total_time_in_millis":125},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":7364,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":5591,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":2,"total_time_in_millis":125},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":7364,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, "2.4.5": `{"_shards":{"total":20,"successful":10,"failed":0},"_all":{"primaries":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":3610,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":40,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":171},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":10530,"terms_memory_in_bytes":7550,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103887660,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":843},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":3610,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":40,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":171},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":10530,"terms_memory_in_bytes":7550,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103887660,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":843},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":3350,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":34},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":470},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":3350,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":34},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":470},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":34,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":137},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4212,"terms_memory_in_bytes":3020,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":373},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":34,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":137},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4212,"terms_memory_in_bytes":3020,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":373},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, "5.4.2": `{"_shards":{"total":26,"successful":13,"failed":0},"_all":{"primaries":{"docs":{"count":76,"deleted":0},"store":{"size_in_bytes":128534,"throttle_time_in_millis":0},"indexing":{"index_total":78,"index_time_in_millis":1598,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":272629760},"refresh":{"total":15,"total_time_in_millis":1361,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":26,"total_time_in_millis":124},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":13,"memory_in_bytes":56523,"terms_memory_in_bytes":44419,"stored_fields_memory_in_bytes":4056,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":2880,"points_memory_in_bytes":652,"doc_values_memory_in_bytes":4516,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":78,"size_in_bytes":56679},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":76,"deleted":0},"store":{"size_in_bytes":128534,"throttle_time_in_millis":0},"indexing":{"index_total":78,"index_time_in_millis":1598,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":272629760},"refresh":{"total":15,"total_time_in_millis":1361,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":26,"total_time_in_millis":124},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":13,"memory_in_bytes":56523,"terms_memory_in_bytes":44419,"stored_fields_memory_in_bytes":4056,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":2880,"points_memory_in_bytes":652,"doc_values_memory_in_bytes":4516,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":78,"size_in_bytes":56679},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-es-2-2017.08.23":{"primaries":{"docs":{"count":65,"deleted":0},"store":{"size_in_bytes":68917,"throttle_time_in_millis":0},"indexing":{"index_total":65,"index_time_in_millis":106,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":390,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":15},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":23830,"terms_memory_in_bytes":18474,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":320,"points_memory_in_bytes":648,"doc_values_memory_in_bytes":3452,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":65,"size_in_bytes":37990},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":65,"deleted":0},"store":{"size_in_bytes":68917,"throttle_time_in_millis":0},"indexing":{"index_total":65,"index_time_in_millis":106,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":390,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":15},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":23830,"terms_memory_in_bytes":18474,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":320,"points_memory_in_bytes":648,"doc_values_memory_in_bytes":3452,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":65,"size_in_bytes":37990},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-data-2":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":4226,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":74,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":3,"total_time_in_millis":2},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1335,"terms_memory_in_bytes":787,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":236,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":6738},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":4226,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":74,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":3,"total_time_in_millis":2},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1335,"terms_memory_in_bytes":787,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":236,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":6738},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11909,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":12,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":42,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":494},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11909,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":12,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":42,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":494},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8038,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":84,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":7,"total_time_in_millis":94},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":2,"size_in_bytes":389},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8038,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":84,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":7,"total_time_in_millis":94},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":2,"size_in_bytes":389},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".watches":{"primaries":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":35444,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":1421,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":5,"total_time_in_millis":771,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":9},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":18418,"terms_memory_in_bytes":15198,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":4,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":11068},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":35444,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":1421,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":5,"total_time_in_millis":771,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":9},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":18418,"terms_memory_in_bytes":15198,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":4,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":11068},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, "5.5.2": `{"_shards":{"total":34,"successful":34,"failed":0},"_all":{"primaries":{"docs":{"count":256658,"deleted":148},"store":{"size_in_bytes":175959144,"throttle_time_in_millis":0},"indexing":{"index_total":413652,"index_time_in_millis":316231,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88458,"query_time_in_millis":2513,"query_current":0,"fetch_total":88446,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":10275,"total_time_in_millis":5895937,"total_docs":80376844,"total_size_in_bytes":49005089428,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":354609338},"refresh":{"total":100264,"total_time_in_millis":818781,"listeners":0},"flush":{"total":5,"total_time_in_millis":124},"warmer":{"current":0,"total":100286,"total_time_in_millis":23851},"query_cache":{"memory_size_in_bytes":0,"total_count":22,"hit_count":0,"miss_count":22,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":38,"memory_in_bytes":720166,"terms_memory_in_bytes":481188,"stored_fields_memory_in_bytes":31920,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":3072,"points_memory_in_bytes":61858,"doc_values_memory_in_bytes":142128,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":137331,"size_in_bytes":167642357},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":513316,"deleted":324},"store":{"size_in_bytes":352143858,"throttle_time_in_millis":0},"indexing":{"index_total":827302,"index_time_in_millis":625030,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176913,"query_time_in_millis":5115,"query_current":0,"fetch_total":176891,"fetch_time_in_millis":5862,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":20547,"total_time_in_millis":11765629,"total_docs":160766745,"total_size_in_bytes":98024993696,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":709218676},"refresh":{"total":200378,"total_time_in_millis":1620051,"listeners":0},"flush":{"total":10,"total_time_in_millis":264},"warmer":{"current":0,"total":200427,"total_time_in_millis":46689},"query_cache":{"memory_size_in_bytes":0,"total_count":54,"hit_count":0,"miss_count":54,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":776,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":86,"memory_in_bytes":1496932,"terms_memory_in_bytes":1008932,"stored_fields_memory_in_bytes":66896,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":6784,"points_memory_in_bytes":124368,"doc_values_memory_in_bytes":289952,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":274662,"size_in_bytes":335284714},"request_cache":{"memory_size_in_bytes":1794,"evictions":0,"hit_count":0,"miss_count":3},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-kibana-6-2017.08.16":{"primaries":{"docs":{"count":12459,"deleted":0},"store":{"size_in_bytes":4880578,"throttle_time_in_millis":0},"indexing":{"index_total":12459,"index_time_in_millis":20073,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1372,"total_time_in_millis":275560,"total_docs":8619963,"total_size_in_bytes":3518738895,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":12724,"total_time_in_millis":79600,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12725,"total_time_in_millis":2653},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":34337,"terms_memory_in_bytes":30104,"stored_fields_memory_in_bytes":968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":1393,"doc_values_memory_in_bytes":1744,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":12459,"size_in_bytes":12175093},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":24918,"deleted":0},"store":{"size_in_bytes":9682148,"throttle_time_in_millis":0},"indexing":{"index_total":24918,"index_time_in_millis":37852,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2743,"total_time_in_millis":549689,"total_docs":17234974,"total_size_in_bytes":7034624631,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":25390,"total_time_in_millis":155546,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":25392,"total_time_in_millis":5196},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83565,"terms_memory_in_bytes":73488,"stored_fields_memory_in_bytes":2968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":2897,"doc_values_memory_in_bytes":3764,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":24918,"size_in_bytes":24350186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.15":{"primaries":{"docs":{"count":25917,"deleted":0},"store":{"size_in_bytes":9617967,"throttle_time_in_millis":0},"indexing":{"index_total":25917,"index_time_in_millis":36304,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2055,"total_time_in_millis":746516,"total_docs":27797269,"total_size_in_bytes":10672726928,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":19569,"total_time_in_millis":123083,"listeners":0},"flush":{"total":1,"total_time_in_millis":18},"warmer":{"current":0,"total":19571,"total_time_in_millis":3995},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":58226,"terms_memory_in_bytes":52997,"stored_fields_memory_in_bytes":1912,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":2721,"doc_values_memory_in_bytes":404,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":51834,"deleted":0},"store":{"size_in_bytes":19309686,"throttle_time_in_millis":0},"indexing":{"index_total":51834,"index_time_in_millis":68258,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":4109,"total_time_in_millis":1477106,"total_docs":55598675,"total_size_in_bytes":21347372840,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":39103,"total_time_in_millis":238886,"listeners":0},"flush":{"total":2,"total_time_in_millis":44},"warmer":{"current":0,"total":39108,"total_time_in_millis":7775},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":141204,"terms_memory_in_bytes":128132,"stored_fields_memory_in_bytes":5472,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":5628,"doc_values_memory_in_bytes":1268,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.14":{"primaries":{"docs":{"count":6147,"deleted":0},"store":{"size_in_bytes":2467088,"throttle_time_in_millis":0},"indexing":{"index_total":6147,"index_time_in_millis":9228,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":397,"total_time_in_millis":48142,"total_docs":1231276,"total_size_in_bytes":559560922,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3733,"total_time_in_millis":28841,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":3735,"total_time_in_millis":1168},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":29099,"terms_memory_in_bytes":25724,"stored_fields_memory_in_bytes":1104,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":715,"doc_values_memory_in_bytes":1364,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":12294,"deleted":0},"store":{"size_in_bytes":5029190,"throttle_time_in_millis":0},"indexing":{"index_total":12294,"index_time_in_millis":16557,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":792,"total_time_in_millis":96523,"total_docs":2467549,"total_size_in_bytes":1121117660,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":7449,"total_time_in_millis":51544,"listeners":0},"flush":{"total":2,"total_time_in_millis":16},"warmer":{"current":0,"total":7453,"total_time_in_millis":1872},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":83924,"terms_memory_in_bytes":73590,"stored_fields_memory_in_bytes":3768,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":1618,"doc_values_memory_in_bytes":4244,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".kibana":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3967,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88441,"query_time_in_millis":2287,"query_current":0,"fetch_total":88441,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":19,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":5,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2219,"terms_memory_in_bytes":1751,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":64,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":7934,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176879,"query_time_in_millis":4703,"query_current":0,"fetch_total":176879,"fetch_time_in_millis":5860,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":4,"total_time_in_millis":33,"listeners":0},"flush":{"total":2,"total_time_in_millis":14},"warmer":{"current":0,"total":10,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4438,"terms_memory_in_bytes":3502,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.15":{"primaries":{"docs":{"count":120970,"deleted":60},"store":{"size_in_bytes":90471073,"throttle_time_in_millis":0},"indexing":{"index_total":207360,"index_time_in_millis":142584,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":85,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3781,"total_time_in_millis":2188959,"total_docs":19326607,"total_size_in_bytes":15981823826,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":19065018},"refresh":{"total":37364,"total_time_in_millis":337011,"listeners":0},"flush":{"total":1,"total_time_in_millis":67},"warmer":{"current":0,"total":37365,"total_time_in_millis":8943},"query_cache":{"memory_size_in_bytes":0,"total_count":9,"hit_count":0,"miss_count":9,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":297633,"terms_memory_in_bytes":184606,"stored_fields_memory_in_bytes":13736,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":31279,"doc_values_memory_in_bytes":67436,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":241940,"deleted":120},"store":{"size_in_bytes":180988889,"throttle_time_in_millis":0},"indexing":{"index_total":414720,"index_time_in_millis":286230,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":135,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":7561,"total_time_in_millis":4425176,"total_docs":38722769,"total_size_in_bytes":32018268991,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":38130036},"refresh":{"total":74706,"total_time_in_millis":675900,"listeners":0},"flush":{"total":2,"total_time_in_millis":148},"warmer":{"current":0,"total":74710,"total_time_in_millis":18069},"query_cache":{"memory_size_in_bytes":0,"total_count":24,"hit_count":0,"miss_count":24,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":424,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":16,"memory_in_bytes":590531,"terms_memory_in_bytes":362031,"stored_fields_memory_in_bytes":26728,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1024,"points_memory_in_bytes":62748,"doc_values_memory_in_bytes":138000,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755201903,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.14":{"primaries":{"docs":{"count":24614,"deleted":12},"store":{"size_in_bytes":17680221,"throttle_time_in_millis":0},"indexing":{"index_total":36896,"index_time_in_millis":35441,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":24,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":927,"total_time_in_millis":1245791,"total_docs":11392506,"total_size_in_bytes":8435013687,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9432,"total_time_in_millis":90529,"listeners":0},"flush":{"total":1,"total_time_in_millis":25},"warmer":{"current":0,"total":9434,"total_time_in_millis":3098},"query_cache":{"memory_size_in_bytes":0,"total_count":2,"hit_count":0,"miss_count":2,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83467,"terms_memory_in_bytes":59601,"stored_fields_memory_in_bytes":4040,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":7054,"doc_values_memory_in_bytes":12324,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":49228,"deleted":18},"store":{"size_in_bytes":35172184,"throttle_time_in_millis":0},"indexing":{"index_total":73792,"index_time_in_millis":69939,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":61,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1853,"total_time_in_millis":2468444,"total_docs":22769560,"total_size_in_bytes":16842830754,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":18843,"total_time_in_millis":176757,"listeners":0},"flush":{"total":2,"total_time_in_millis":42},"warmer":{"current":0,"total":18847,"total_time_in_millis":5819},"query_cache":{"memory_size_in_bytes":0,"total_count":7,"hit_count":0,"miss_count":7,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":352,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":137869,"terms_memory_in_bytes":97131,"stored_fields_memory_in_bytes":6680,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":13662,"doc_values_memory_in_bytes":19820,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":12138,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":5,"query_time_in_millis":0,"query_current":0,"fetch_total":2,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":12,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":524},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":6,"deleted":0},"store":{"size_in_bytes":24276,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10,"query_time_in_millis":0,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":20,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":6,"memory_in_bytes":15528,"terms_memory_in_bytes":11952,"stored_fields_memory_in_bytes":1872,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1152,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":552,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":6,"size_in_bytes":1048},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8246,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":7,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":7,"query_time_in_millis":8,"query_current":0,"fetch_total":3,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":4,"total_time_in_millis":18,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":9,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":609},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":16492,"throttle_time_in_millis":0},"indexing":{"index_total":7,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":15,"query_time_in_millis":21,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":7,"total_time_in_millis":32,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":10352,"terms_memory_in_bytes":7968,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":768,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":8,"size_in_bytes":1218},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.16":{"primaries":{"docs":{"count":66545,"deleted":76},"store":{"size_in_bytes":50817866,"throttle_time_in_millis":0},"indexing":{"index_total":124865,"index_time_in_millis":72588,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":67,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1743,"total_time_in_millis":1390969,"total_docs":12009223,"total_size_in_bytes":9837225170,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":17433,"total_time_in_millis":159668,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":17434,"total_time_in_millis":3982},"query_cache":{"memory_size_in_bytes":0,"total_count":8,"hit_count":0,"miss_count":8,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":8,"memory_in_bytes":202245,"terms_memory_in_bytes":116445,"stored_fields_memory_in_bytes":8288,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":512,"points_memory_in_bytes":18696,"doc_values_memory_in_bytes":58304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":124865,"size_in_bytes":155465916},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":133090,"deleted":186},"store":{"size_in_bytes":101913059,"throttle_time_in_millis":0},"indexing":{"index_total":249730,"index_time_in_millis":146171,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":143,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3489,"total_time_in_millis":2748691,"total_docs":23973218,"total_size_in_bytes":19660778820,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":34871,"total_time_in_millis":321333,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":34873,"total_time_in_millis":7932},"query_cache":{"memory_size_in_bytes":0,"total_count":17,"hit_count":0,"miss_count":17,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":20,"memory_in_bytes":429521,"terms_memory_in_bytes":251138,"stored_fields_memory_in_bytes":17536,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1280,"points_memory_in_bytes":37815,"doc_values_memory_in_bytes":121752,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":249730,"size_in_bytes":310931832},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, } for ver, out := range ti { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, out) })) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("Failed to parse URL: %s", err) } i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, false) stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) } t.Logf("[%s] Index Response: %+v", ver, stats) if stats.Indices["foo_1"].Primaries.Docs.Count != 2 { t.Errorf("Wrong number of primary docs") } if stats.Indices["foo_1"].Primaries.Store.SizeInBytes == 0 { t.Errorf("Wrong number of primary store size in bytes") } if stats.Indices["foo_1"].Total.Store.SizeInBytes == 0 { t.Errorf("Wrong number of total store size in bytes") } if stats.Indices["foo_1"].Total.Indexing.IndexTimeInMillis == 0 { t.Errorf("Wrong indexing time recorded") } if stats.Indices["foo_1"].Total.Indexing.IndexTotal == 0 { t.Errorf("Wrong indexing total recorded") } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/nodes.go000066400000000000000000001700621362377146000244300ustar00rootroot00000000000000package collector import ( "encoding/json" "fmt" "net/http" "net/url" "path" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) func getRoles(node NodeStatsNodeResponse) map[string]bool { // default settings (2.x) and map, which roles to consider roles := map[string]bool{ "master": false, "data": false, "ingest": false, "client": true, } // assumption: a 5.x node has at least one role, otherwise it's a 1.7 or 2.x node if len(node.Roles) > 0 { for _, role := range node.Roles { // set every absent role to false if _, ok := roles[role]; !ok { roles[role] = false } else { // if present in the roles field, set to true roles[role] = true } } } else { for role, setting := range node.Attributes { if _, ok := roles[role]; ok { if setting == "false" { roles[role] = false } else { roles[role] = true } } } } if len(node.HTTP) == 0 { roles["client"] = false } return roles } func createRoleMetric(role string) *nodeMetric { return &nodeMetric{ Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "nodes", "roles"), "Node roles", defaultRoleLabels, prometheus.Labels{"role": role}, ), Value: func(node NodeStatsNodeResponse) float64 { return 1.0 }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return []string{ cluster, node.Host, node.Name, } }, } } var ( defaultNodeLabels = []string{"cluster", "host", "name", "es_master_node", "es_data_node", "es_ingest_node", "es_client_node"} defaultRoleLabels = []string{"cluster", "host", "name"} defaultThreadPoolLabels = append(defaultNodeLabels, "type") defaultBreakerLabels = append(defaultNodeLabels, "breaker") defaultFilesystemDataLabels = append(defaultNodeLabels, "mount", "path") defaultFilesystemIODeviceLabels = append(defaultNodeLabels, "device") defaultCacheLabels = append(defaultNodeLabels, "cache") defaultNodeLabelValues = func(cluster string, node NodeStatsNodeResponse) []string { roles := getRoles(node) return []string{ cluster, node.Host, node.Name, fmt.Sprintf("%t", roles["master"]), fmt.Sprintf("%t", roles["data"]), fmt.Sprintf("%t", roles["ingest"]), fmt.Sprintf("%t", roles["client"]), } } defaultThreadPoolLabelValues = func(cluster string, node NodeStatsNodeResponse, pool string) []string { return append(defaultNodeLabelValues(cluster, node), pool) } defaultFilesystemDataLabelValues = func(cluster string, node NodeStatsNodeResponse, mount string, path string) []string { return append(defaultNodeLabelValues(cluster, node), mount, path) } defaultFilesystemIODeviceLabelValues = func(cluster string, node NodeStatsNodeResponse, device string) []string { return append(defaultNodeLabelValues(cluster, node), device) } defaultCacheHitLabelValues = func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "hit") } defaultCacheMissLabelValues = func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "miss") } ) type nodeMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(node NodeStatsNodeResponse) float64 Labels func(cluster string, node NodeStatsNodeResponse) []string } type gcCollectionMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(gcStats NodeStatsJVMGCCollectorResponse) float64 Labels func(cluster string, node NodeStatsNodeResponse, collector string) []string } type breakerMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(breakerStats NodeStatsBreakersResponse) float64 Labels func(cluster string, node NodeStatsNodeResponse, breaker string) []string } type threadPoolMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 Labels func(cluster string, node NodeStatsNodeResponse, breaker string) []string } type filesystemDataMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(fsStats NodeStatsFSDataResponse) float64 Labels func(cluster string, node NodeStatsNodeResponse, mount string, path string) []string } type filesystemIODeviceMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(fsStats NodeStatsFSIOStatsDeviceResponse) float64 Labels func(cluster string, node NodeStatsNodeResponse, device string) []string } // Nodes information struct type Nodes struct { logger log.Logger client *http.Client url *url.URL all bool node string up prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter nodeMetrics []*nodeMetric gcCollectionMetrics []*gcCollectionMetric breakerMetrics []*breakerMetric threadPoolMetrics []*threadPoolMetric filesystemDataMetrics []*filesystemDataMetric filesystemIODeviceMetrics []*filesystemIODeviceMetric } // NewNodes defines Nodes Prometheus metrics func NewNodes(logger log.Logger, client *http.Client, url *url.URL, all bool, node string) *Nodes { return &Nodes{ logger: logger, client: client, url: url, all: all, node: node, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "node_stats", "up"), Help: "Was the last scrape of the ElasticSearch nodes endpoint successful.", }), totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "node_stats", "total_scrapes"), Help: "Current total ElasticSearch node scrapes.", }), jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "node_stats", "json_parse_failures"), Help: "Number of errors while parsing JSON.", }), nodeMetrics: []*nodeMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "load1"), "Shortterm load average", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return node.OS.CPU.LoadAvg.Load1 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "load5"), "Midterm load average", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return node.OS.CPU.LoadAvg.Load5 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "load15"), "Longterm load average", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return node.OS.CPU.LoadAvg.Load15 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "cpu_percent"), "Percent CPU used by OS", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.OS.CPU.Percent) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "mem_free_bytes"), "Amount of free physical memory in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.OS.Mem.Free) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "mem_used_bytes"), "Amount of used physical memory in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.OS.Mem.Used) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "mem_actual_free_bytes"), "Amount of free physical memory in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.OS.Mem.ActualFree) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "os", "mem_actual_used_bytes"), "Amount of used physical memory in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.OS.Mem.ActualUsed) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "fielddata_memory_size_bytes"), "Field data cache memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.FieldData.MemorySize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "fielddata_evictions"), "Evictions from field data", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.FieldData.Evictions) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "completion_size_in_bytes"), "Completion in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Completion.Size) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "filter_cache_memory_size_bytes"), "Filter cache memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.FilterCache.MemorySize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "filter_cache_evictions"), "Evictions from filter cache", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.FilterCache.Evictions) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_cache_memory_size_bytes"), "Query cache memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.MemorySize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_cache_evictions"), "Evictions from query cache", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.Evictions) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_cache_total"), "Query cache total count", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.TotalCount) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_cache_cache_size"), "Query cache cache size", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.CacheSize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_cache_cache_total"), "Query cache cache count", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.CacheCount) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_cache_count"), "Query cache count", defaultCacheLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.HitCount) }, Labels: defaultCacheHitLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "query_miss_count"), "Query miss count", defaultCacheLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.QueryCache.MissCount) }, Labels: defaultCacheMissLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "request_cache_memory_size_bytes"), "Request cache memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.RequestCache.MemorySize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "request_cache_evictions"), "Evictions from request cache", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.RequestCache.Evictions) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "request_cache_count"), "Request cache count", defaultCacheLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.RequestCache.HitCount) }, Labels: defaultCacheHitLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "request_miss_count"), "Request miss count", defaultCacheLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.RequestCache.MissCount) }, Labels: defaultCacheMissLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "translog_operations"), "Total translog operations", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Translog.Operations) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "translog_size_in_bytes"), "Total translog size in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Translog.Size) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "get_time_seconds"), "Total get time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Get.Time) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "get_total"), "Total get", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Get.Total) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "get_missing_time_seconds"), "Total time of get missing in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Get.MissingTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "get_missing_total"), "Total get missing", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Get.MissingTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "get_exists_time_seconds"), "Total time get exists in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Get.ExistsTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "get_exists_total"), "Total get exists operations", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Get.ExistsTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_refresh", "time_seconds_total"), "Total time spent refreshing in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Refresh.TotalTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_refresh", "total"), "Total refreshes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Refresh.Total) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_query_time_seconds"), "Total search query time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.QueryTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_query_total"), "Total number of queries", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.QueryTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_fetch_time_seconds"), "Total search fetch time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.FetchTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_fetch_total"), "Total number of fetches", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.FetchTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_suggest_total"), "Total number of suggests", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.SuggestTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_suggest_time_seconds"), "Total suggest time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.SuggestTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_scroll_total"), "Total number of scrolls", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.ScrollTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "search_scroll_time_seconds"), "Total scroll time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Search.ScrollTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "docs"), "Count of documents on this node", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Docs.Count) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "docs_deleted"), "Count of deleted documents on this node", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Docs.Deleted) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "store_size_bytes"), "Current size of stored index data in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Store.Size) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "store_throttle_time_seconds_total"), "Throttle time for index store in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Store.ThrottleTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_memory_bytes"), "Current memory size of segments in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.Memory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_count"), "Count of index segments on this node", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.Count) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_terms_memory_in_bytes"), "Count of terms in memory for this node", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.TermsMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_index_writer_memory_in_bytes"), "Count of memory for index writer on this node", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.IndexWriterMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_norms_memory_in_bytes"), "Count of memory used by norms", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.NormsMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_stored_fields_memory_in_bytes"), "Count of stored fields memory", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.StoredFieldsMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_doc_values_memory_in_bytes"), "Count of doc values memory", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.DocValuesMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_fixed_bit_set_memory_in_bytes"), "Count of fixed bit set", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.FixedBitSet) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_term_vectors_memory_in_bytes"), "Term vectors memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.TermVectorsMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_points_memory_in_bytes"), "Point values memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.PointsMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "segments_version_map_memory_in_bytes"), "Version map memory usage in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Segments.VersionMapMemory) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "flush_total"), "Total flushes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Flush.Total) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "flush_time_seconds"), "Cumulative flush time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Flush.Time) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "warmer_total"), "Total warmer count", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Warmer.Total) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices", "warmer_time_seconds_total"), "Total warmer time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Warmer.TotalTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_indexing", "index_time_seconds_total"), "Cumulative index time in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Indexing.IndexTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_indexing", "index_total"), "Total index calls", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Indexing.IndexTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_indexing", "delete_time_seconds_total"), "Total time indexing delete in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Indexing.DeleteTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_indexing", "delete_total"), "Total indexing deletes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Indexing.DeleteTotal) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_indexing", "is_throttled"), "Indexing throttling", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { if node.Indices.Indexing.IsThrottled { return 1 } return 0 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_indexing", "throttle_time_seconds_total"), "Cumulative indexing throttling time", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Indexing.ThrottleTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "total"), "Total merges", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.Total) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "current"), "Current merges", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.Current) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "current_size_in_bytes"), "Size of a current merges in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.CurrentSize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "docs_total"), "Cumulative docs merged", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.TotalDocs) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "total_size_bytes_total"), "Total merge size in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.TotalSize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "total_time_seconds_total"), "Total time spent merging in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.TotalTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "indices_merges", "total_throttled_time_seconds_total"), "Total throttled time of merges in seconds", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Indices.Merges.TotalThrottledTime) / 1000 }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory", "used_bytes"), "JVM memory currently used by area", append(defaultNodeLabels, "area"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.HeapUsed) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "heap") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory", "used_bytes"), "JVM memory currently used by area", append(defaultNodeLabels, "area"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.NonHeapUsed) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "non-heap") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory", "max_bytes"), "JVM memory max", append(defaultNodeLabels, "area"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.HeapMax) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "heap") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory", "committed_bytes"), "JVM memory currently committed by area", append(defaultNodeLabels, "area"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.HeapCommitted) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "heap") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory", "committed_bytes"), "JVM memory currently committed by area", append(defaultNodeLabels, "area"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.NonHeapCommitted) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "non-heap") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "used_bytes"), "JVM memory currently used by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["young"].Used) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "young") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "max_bytes"), "JVM memory max by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["young"].Max) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "young") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "peak_used_bytes"), "JVM memory peak used by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["young"].PeakUsed) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "young") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "peak_max_bytes"), "JVM memory peak max by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["young"].PeakMax) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "young") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "used_bytes"), "JVM memory currently used by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["survivor"].Used) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "survivor") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "max_bytes"), "JVM memory max by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["survivor"].Max) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "survivor") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "peak_used_bytes"), "JVM memory peak used by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["survivor"].PeakUsed) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "survivor") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "peak_max_bytes"), "JVM memory peak max by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["survivor"].PeakMax) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "survivor") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "used_bytes"), "JVM memory currently used by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["old"].Used) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "old") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "max_bytes"), "JVM memory max by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["old"].Max) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "old") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "peak_used_bytes"), "JVM memory peak used by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["old"].PeakUsed) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "old") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_memory_pool", "peak_max_bytes"), "JVM memory peak max by pool", append(defaultNodeLabels, "pool"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.Mem.Pools["old"].PeakMax) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "old") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_buffer_pool", "used_bytes"), "JVM buffer currently used", append(defaultNodeLabels, "type"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.BufferPools["direct"].Used) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "direct") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_buffer_pool", "used_bytes"), "JVM buffer currently used", append(defaultNodeLabels, "type"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.JVM.BufferPools["mapped"].Used) }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "mapped") }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "cpu_percent"), "Percent CPU used by process", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.CPU.Percent) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "mem_resident_size_bytes"), "Resident memory in use by process in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.Memory.Resident) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "mem_share_size_bytes"), "Shared memory in use by process in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.Memory.Share) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "mem_virtual_size_bytes"), "Total virtual memory used in bytes", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.Memory.TotalVirtual) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "open_files_count"), "Open file descriptors", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.OpenFD) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "max_files_descriptors"), "Max file descriptors", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.MaxFD) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "cpu_time_seconds_sum"), "Process CPU time in seconds", append(defaultNodeLabels, "type"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.CPU.Total) / 1000 }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "total") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "cpu_time_seconds_sum"), "Process CPU time in seconds", append(defaultNodeLabels, "type"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.CPU.Sys) / 1000 }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "sys") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "process", "cpu_time_seconds_sum"), "Process CPU time in seconds", append(defaultNodeLabels, "type"), nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Process.CPU.User) / 1000 }, Labels: func(cluster string, node NodeStatsNodeResponse) []string { return append(defaultNodeLabelValues(cluster, node), "user") }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "transport", "rx_packets_total"), "Count of packets received", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Transport.RxCount) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "transport", "rx_size_bytes_total"), "Total number of bytes received", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Transport.RxSize) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "transport", "tx_packets_total"), "Count of packets sent", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Transport.TxCount) }, Labels: defaultNodeLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "transport", "tx_size_bytes_total"), "Total number of bytes sent", defaultNodeLabels, nil, ), Value: func(node NodeStatsNodeResponse) float64 { return float64(node.Transport.TxSize) }, Labels: defaultNodeLabelValues, }, }, gcCollectionMetrics: []*gcCollectionMetric{ { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_gc", "collection_seconds_count"), "Count of JVM GC runs", append(defaultNodeLabels, "gc"), nil, ), Value: func(gcStats NodeStatsJVMGCCollectorResponse) float64 { return float64(gcStats.CollectionCount) }, Labels: func(cluster string, node NodeStatsNodeResponse, collector string) []string { return append(defaultNodeLabelValues(cluster, node), collector) }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "jvm_gc", "collection_seconds_sum"), "GC run time in seconds", append(defaultNodeLabels, "gc"), nil, ), Value: func(gcStats NodeStatsJVMGCCollectorResponse) float64 { return float64(gcStats.CollectionTime) / 1000 }, Labels: func(cluster string, node NodeStatsNodeResponse, collector string) []string { return append(defaultNodeLabelValues(cluster, node), collector) }, }, }, breakerMetrics: []*breakerMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "breakers", "estimated_size_bytes"), "Estimated size in bytes of breaker", defaultBreakerLabels, nil, ), Value: func(breakerStats NodeStatsBreakersResponse) float64 { return float64(breakerStats.EstimatedSize) }, Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { return append(defaultNodeLabelValues(cluster, node), breaker) }, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "breakers", "limit_size_bytes"), "Limit size in bytes for breaker", defaultBreakerLabels, nil, ), Value: func(breakerStats NodeStatsBreakersResponse) float64 { return float64(breakerStats.LimitSize) }, Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { return append(defaultNodeLabelValues(cluster, node), breaker) }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "breakers", "tripped"), "tripped for breaker", defaultBreakerLabels, nil, ), Value: func(breakerStats NodeStatsBreakersResponse) float64 { return float64(breakerStats.Tripped) }, Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { return append(defaultNodeLabelValues(cluster, node), breaker) }, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "breakers", "overhead"), "Overhead of circuit breakers", defaultBreakerLabels, nil, ), Value: func(breakerStats NodeStatsBreakersResponse) float64 { return breakerStats.Overhead }, Labels: func(cluster string, node NodeStatsNodeResponse, breaker string) []string { return append(defaultNodeLabelValues(cluster, node), breaker) }, }, }, threadPoolMetrics: []*threadPoolMetric{ { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "thread_pool", "completed_count"), "Thread Pool operations completed", defaultThreadPoolLabels, nil, ), Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { return float64(threadPoolStats.Completed) }, Labels: defaultThreadPoolLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "thread_pool", "rejected_count"), "Thread Pool operations rejected", defaultThreadPoolLabels, nil, ), Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { return float64(threadPoolStats.Rejected) }, Labels: defaultThreadPoolLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "thread_pool", "active_count"), "Thread Pool threads active", defaultThreadPoolLabels, nil, ), Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { return float64(threadPoolStats.Active) }, Labels: defaultThreadPoolLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "thread_pool", "largest_count"), "Thread Pool largest threads count", defaultThreadPoolLabels, nil, ), Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { return float64(threadPoolStats.Largest) }, Labels: defaultThreadPoolLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "thread_pool", "queue_count"), "Thread Pool operations queued", defaultThreadPoolLabels, nil, ), Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { return float64(threadPoolStats.Queue) }, Labels: defaultThreadPoolLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "thread_pool", "threads_count"), "Thread Pool current threads count", defaultThreadPoolLabels, nil, ), Value: func(threadPoolStats NodeStatsThreadPoolPoolResponse) float64 { return float64(threadPoolStats.Threads) }, Labels: defaultThreadPoolLabelValues, }, }, filesystemDataMetrics: []*filesystemDataMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_data", "available_bytes"), "Available space on block device in bytes", defaultFilesystemDataLabels, nil, ), Value: func(fsStats NodeStatsFSDataResponse) float64 { return float64(fsStats.Available) }, Labels: defaultFilesystemDataLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_data", "free_bytes"), "Free space on block device in bytes", defaultFilesystemDataLabels, nil, ), Value: func(fsStats NodeStatsFSDataResponse) float64 { return float64(fsStats.Free) }, Labels: defaultFilesystemDataLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_data", "size_bytes"), "Size of block device in bytes", defaultFilesystemDataLabels, nil, ), Value: func(fsStats NodeStatsFSDataResponse) float64 { return float64(fsStats.Total) }, Labels: defaultFilesystemDataLabelValues, }, }, filesystemIODeviceMetrics: []*filesystemIODeviceMetric{ { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "operations_count"), "Count of disk operations", defaultFilesystemIODeviceLabels, nil, ), Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { return float64(fsIODeviceStats.Operations) }, Labels: defaultFilesystemIODeviceLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "read_operations_count"), "Count of disk read operations", defaultFilesystemIODeviceLabels, nil, ), Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { return float64(fsIODeviceStats.ReadOperations) }, Labels: defaultFilesystemIODeviceLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "write_operations_count"), "Count of disk write operations", defaultFilesystemIODeviceLabels, nil, ), Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { return float64(fsIODeviceStats.WriteOperations) }, Labels: defaultFilesystemIODeviceLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "read_size_kilobytes_sum"), "Total kilobytes read from disk", defaultFilesystemIODeviceLabels, nil, ), Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { return float64(fsIODeviceStats.ReadSize) }, Labels: defaultFilesystemIODeviceLabelValues, }, { Type: prometheus.CounterValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filesystem_io_stats_device", "write_size_kilobytes_sum"), "Total kilobytes written to disk", defaultFilesystemIODeviceLabels, nil, ), Value: func(fsIODeviceStats NodeStatsFSIOStatsDeviceResponse) float64 { return float64(fsIODeviceStats.WriteSize) }, Labels: defaultFilesystemIODeviceLabelValues, }, }, } } // Describe add metrics descriptions func (c *Nodes) Describe(ch chan<- *prometheus.Desc) { for _, metric := range c.nodeMetrics { ch <- metric.Desc } for _, metric := range c.gcCollectionMetrics { ch <- metric.Desc } for _, metric := range c.threadPoolMetrics { ch <- metric.Desc } for _, metric := range c.filesystemDataMetrics { ch <- metric.Desc } for _, metric := range c.filesystemIODeviceMetrics { ch <- metric.Desc } ch <- c.up.Desc() ch <- c.totalScrapes.Desc() ch <- c.jsonParseFailures.Desc() } func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { var nsr nodeStatsResponse u := *c.url if c.all { u.Path = path.Join(u.Path, "/_nodes/stats") } else { u.Path = path.Join(u.Path, "_nodes", c.node, "stats") } res, err := c.client.Get(u.String()) if err != nil { return nsr, fmt.Errorf("failed to get cluster health from %s://%s:%s%s: %s", u.Scheme, u.Hostname(), u.Port(), u.Path, err) } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(c.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return nsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(&nsr); err != nil { c.jsonParseFailures.Inc() return nsr, err } return nsr, nil } // Collect gets nodes metric values func (c *Nodes) Collect(ch chan<- prometheus.Metric) { c.totalScrapes.Inc() defer func() { ch <- c.up ch <- c.totalScrapes ch <- c.jsonParseFailures }() nodeStatsResp, err := c.fetchAndDecodeNodeStats() if err != nil { c.up.Set(0) _ = level.Warn(c.logger).Log( "msg", "failed to fetch and decode node stats", "err", err, ) return } c.up.Set(1) for _, node := range nodeStatsResp.Nodes { // Handle the node labels metric roles := getRoles(node) for _, role := range []string{"master", "data", "client", "ingest"} { if roles[role] { metric := createRoleMetric(role) ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(node), metric.Labels(nodeStatsResp.ClusterName, node)..., ) } } for _, metric := range c.nodeMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(node), metric.Labels(nodeStatsResp.ClusterName, node)..., ) } // GC Stats for collector, gcStats := range node.JVM.GC.Collectors { for _, metric := range c.gcCollectionMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(gcStats), metric.Labels(nodeStatsResp.ClusterName, node, collector)..., ) } } // Breaker stats for breaker, bstats := range node.Breakers { for _, metric := range c.breakerMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(bstats), metric.Labels(nodeStatsResp.ClusterName, node, breaker)..., ) } } // Thread Pool stats for pool, pstats := range node.ThreadPool { for _, metric := range c.threadPoolMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(pstats), metric.Labels(nodeStatsResp.ClusterName, node, pool)..., ) } } // File System Data Stats for _, fsDataStats := range node.FS.Data { for _, metric := range c.filesystemDataMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(fsDataStats), metric.Labels(nodeStatsResp.ClusterName, node, fsDataStats.Mount, fsDataStats.Path)..., ) } } // File System IO Device Stats for _, fsIODeviceStats := range node.FS.IOStats.Devices { for _, metric := range c.filesystemIODeviceMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(fsIODeviceStats), metric.Labels(nodeStatsResp.ClusterName, node, fsIODeviceStats.DeviceName)..., ) } } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/nodes_response.go000066400000000000000000000401121362377146000263360ustar00rootroot00000000000000package collector import "encoding/json" // nodeStatsResponse is a representation of a Elasticsearch Node Stats type nodeStatsResponse struct { ClusterName string `json:"cluster_name"` Nodes map[string]NodeStatsNodeResponse } // NodeStatsNodeResponse defines node stats information structure for nodes type NodeStatsNodeResponse struct { Name string `json:"name"` Host string `json:"host"` Timestamp int64 `json:"timestamp"` TransportAddress string `json:"transport_address"` Hostname string `json:"hostname"` Roles []string `json:"roles"` Attributes map[string]string `json:"attributes"` Indices NodeStatsIndicesResponse `json:"indices"` OS NodeStatsOSResponse `json:"os"` Network NodeStatsNetworkResponse `json:"network"` FS NodeStatsFSResponse `json:"fs"` ThreadPool map[string]NodeStatsThreadPoolPoolResponse `json:"thread_pool"` JVM NodeStatsJVMResponse `json:"jvm"` Breakers map[string]NodeStatsBreakersResponse `json:"breakers"` HTTP map[string]int `json:"http"` Transport NodeStatsTransportResponse `json:"transport"` Process NodeStatsProcessResponse `json:"process"` } // NodeStatsBreakersResponse is a representation of a statistics about the field data circuit breaker type NodeStatsBreakersResponse struct { EstimatedSize int64 `json:"estimated_size_in_bytes"` LimitSize int64 `json:"limit_size_in_bytes"` Overhead float64 `json:"overhead"` Tripped int64 `json:"tripped"` } // NodeStatsJVMResponse is a representation of a JVM stats, memory pool information, garbage collection, buffer pools, number of loaded/unloaded classes type NodeStatsJVMResponse struct { BufferPools map[string]NodeStatsJVMBufferPoolResponse `json:"buffer_pools"` GC NodeStatsJVMGCResponse `json:"gc"` Mem NodeStatsJVMMemResponse `json:"mem"` } // NodeStatsJVMGCResponse defines node stats JVM garbage collector information structure type NodeStatsJVMGCResponse struct { Collectors map[string]NodeStatsJVMGCCollectorResponse `json:"collectors"` } // NodeStatsJVMGCCollectorResponse defines node stats JVM garbage collector collection information structure type NodeStatsJVMGCCollectorResponse struct { CollectionCount int64 `json:"collection_count"` CollectionTime int64 `json:"collection_time_in_millis"` } // NodeStatsJVMBufferPoolResponse defines node stats JVM buffer pool information structure type NodeStatsJVMBufferPoolResponse struct { Count int64 `json:"count"` TotalCapacity int64 `json:"total_capacity_in_bytes"` Used int64 `json:"used_in_bytes"` } // NodeStatsJVMMemResponse defines node stats JVM memory information structure type NodeStatsJVMMemResponse struct { HeapCommitted int64 `json:"heap_committed_in_bytes"` HeapUsed int64 `json:"heap_used_in_bytes"` HeapMax int64 `json:"heap_max_in_bytes"` NonHeapCommitted int64 `json:"non_heap_committed_in_bytes"` NonHeapUsed int64 `json:"non_heap_used_in_bytes"` Pools map[string]NodeStatsJVMMemPoolResponse `json:"pools"` } // NodeStatsJVMMemPoolResponse defines node stats JVM memory pool information structure type NodeStatsJVMMemPoolResponse struct { Used int64 `json:"used_in_bytes"` Max int64 `json:"max_in_bytes"` PeakUsed int64 `json:"peak_used_in_bytes"` PeakMax int64 `json:"peak_max_in_bytes"` } // NodeStatsNetworkResponse defines node stats network information structure type NodeStatsNetworkResponse struct { TCP NodeStatsTCPResponse `json:"tcp"` } // NodeStatsTransportResponse is a representation of a transport statistics about sent and received bytes in cluster communication type NodeStatsTransportResponse struct { ServerOpen int64 `json:"server_open"` RxCount int64 `json:"rx_count"` RxSize int64 `json:"rx_size_in_bytes"` TxCount int64 `json:"tx_count"` TxSize int64 `json:"tx_size_in_bytes"` } // NodeStatsThreadPoolPoolResponse is a representation of a statistics about each thread pool, including current size, queue and rejected tasks type NodeStatsThreadPoolPoolResponse struct { Threads int64 `json:"threads"` Queue int64 `json:"queue"` Active int64 `json:"active"` Rejected int64 `json:"rejected"` Largest int64 `json:"largest"` Completed int64 `json:"completed"` } // NodeStatsTCPResponse defines node stats TCP information structure type NodeStatsTCPResponse struct { ActiveOpens int64 `json:"active_opens"` PassiveOpens int64 `json:"passive_opens"` CurrEstab int64 `json:"curr_estab"` InSegs int64 `json:"in_segs"` OutSegs int64 `json:"out_segs"` RetransSegs int64 `json:"retrans_segs"` EstabResets int64 `json:"estab_resets"` AttemptFails int64 `json:"attempt_fails"` InErrs int64 `json:"in_errs"` OutRsts int64 `json:"out_rsts"` } // NodeStatsIndicesResponse is a representation of a indices stats (size, document count, indexing and deletion times, search times, field cache size, merges and flushes) type NodeStatsIndicesResponse struct { Docs NodeStatsIndicesDocsResponse Store NodeStatsIndicesStoreResponse Indexing NodeStatsIndicesIndexingResponse Merges NodeStatsIndicesMergesResponse Get NodeStatsIndicesGetResponse Search NodeStatsIndicesSearchResponse FieldData NodeStatsIndicesCacheResponse `json:"fielddata"` FilterCache NodeStatsIndicesCacheResponse `json:"filter_cache"` QueryCache NodeStatsIndicesCacheResponse `json:"query_cache"` RequestCache NodeStatsIndicesCacheResponse `json:"request_cache"` Flush NodeStatsIndicesFlushResponse Warmer NodeStatsIndicesWarmerResponse Segments NodeStatsIndicesSegmentsResponse Refresh NodeStatsIndicesRefreshResponse Translog NodeStatsIndicesTranslogResponse Completion NodeStatsIndicesCompletionResponse } // NodeStatsIndicesDocsResponse defines node stats docs information structure for indices type NodeStatsIndicesDocsResponse struct { Count int64 `json:"count"` Deleted int64 `json:"deleted"` } // NodeStatsIndicesRefreshResponse defines node stats refresh information structure for indices type NodeStatsIndicesRefreshResponse struct { Total int64 `json:"total"` TotalTime int64 `json:"total_time_in_millis"` } // NodeStatsIndicesTranslogResponse defines node stats translog information structure for indices type NodeStatsIndicesTranslogResponse struct { Operations int64 `json:"operations"` Size int64 `json:"size_in_bytes"` } // NodeStatsIndicesCompletionResponse defines node stats completion information structure for indices type NodeStatsIndicesCompletionResponse struct { Size int64 `json:"size_in_bytes"` } // NodeStatsIndicesSegmentsResponse defines node stats segments information structure for indices type NodeStatsIndicesSegmentsResponse struct { Count int64 `json:"count"` Memory int64 `json:"memory_in_bytes"` TermsMemory int64 `json:"terms_memory_in_bytes"` IndexWriterMemory int64 `json:"index_writer_memory_in_bytes"` NormsMemory int64 `json:"norms_memory_in_bytes"` StoredFieldsMemory int64 `json:"stored_fields_memory_in_bytes"` FixedBitSet int64 `json:"fixed_bit_set_memory_in_bytes"` DocValuesMemory int64 `json:"doc_values_memory_in_bytes"` TermVectorsMemory int64 `json:"term_vectors_memory_in_bytes"` PointsMemory int64 `json:"points_memory_in_bytes"` VersionMapMemory int64 `json:"version_map_memory_in_bytes"` } // NodeStatsIndicesStoreResponse defines node stats store information structure for indices type NodeStatsIndicesStoreResponse struct { Size int64 `json:"size_in_bytes"` ThrottleTime int64 `json:"throttle_time_in_millis"` } // NodeStatsIndicesIndexingResponse defines node stats indexing information structure for indices type NodeStatsIndicesIndexingResponse struct { IndexTotal int64 `json:"index_total"` IndexTime int64 `json:"index_time_in_millis"` IndexCurrent int64 `json:"index_current"` DeleteTotal int64 `json:"delete_total"` DeleteTime int64 `json:"delete_time_in_millis"` DeleteCurrent int64 `json:"delete_current"` IsThrottled bool `json:"is_throttled"` ThrottleTime int64 `json:"throttle_time_in_millis"` } // NodeStatsIndicesMergesResponse defines node stats merges information structure for indices type NodeStatsIndicesMergesResponse struct { Current int64 `json:"current"` CurrentDocs int64 `json:"current_docs"` CurrentSize int64 `json:"current_size_in_bytes"` Total int64 `json:"total"` TotalDocs int64 `json:"total_docs"` TotalSize int64 `json:"total_size_in_bytes"` TotalTime int64 `json:"total_time_in_millis"` TotalThrottledTime int64 `json:"total_throttled_time_in_millis"` } // NodeStatsIndicesGetResponse defines node stats get information structure for indices type NodeStatsIndicesGetResponse struct { Total int64 `json:"total"` Time int64 `json:"time_in_millis"` ExistsTotal int64 `json:"exists_total"` ExistsTime int64 `json:"exists_time_in_millis"` MissingTotal int64 `json:"missing_total"` MissingTime int64 `json:"missing_time_in_millis"` Current int64 `json:"current"` } // NodeStatsIndicesSearchResponse defines node stats search information structure for indices type NodeStatsIndicesSearchResponse struct { OpenContext int64 `json:"open_contexts"` QueryTotal int64 `json:"query_total"` QueryTime int64 `json:"query_time_in_millis"` QueryCurrent int64 `json:"query_current"` FetchTotal int64 `json:"fetch_total"` FetchTime int64 `json:"fetch_time_in_millis"` FetchCurrent int64 `json:"fetch_current"` SuggestTotal int64 `json:"suggest_total"` SuggestTime int64 `json:"suggest_time_in_millis"` ScrollTotal int64 `json:"scroll_total"` ScrollTime int64 `json:"scroll_time_in_millis"` } // NodeStatsIndicesFlushResponse defines node stats flush information structure for indices type NodeStatsIndicesFlushResponse struct { Total int64 `json:"total"` Time int64 `json:"total_time_in_millis"` } // NodeStatsIndicesWarmerResponse defines node stats warmer information structure for indices type NodeStatsIndicesWarmerResponse struct { Total int64 `json:"total"` TotalTime int64 `json:"total_time_in_millis"` } // NodeStatsIndicesCacheResponse defines node stats cache information structure for indices type NodeStatsIndicesCacheResponse struct { Evictions int64 `json:"evictions"` MemorySize int64 `json:"memory_size_in_bytes"` CacheCount int64 `json:"cache_count"` CacheSize int64 `json:"cache_size"` HitCount int64 `json:"hit_count"` MissCount int64 `json:"miss_count"` TotalCount int64 `json:"total_count"` } // NodeStatsOSResponse is a representation of a operating system stats, load average, mem, swap type NodeStatsOSResponse struct { Timestamp int64 `json:"timestamp"` Uptime int64 `json:"uptime_in_millis"` // LoadAvg was an array of per-cpu values pre-2.0, and is a string in 2.0 // Leaving this here in case we want to implement parsing logic later LoadAvg json.RawMessage `json:"load_average"` CPU NodeStatsOSCPUResponse `json:"cpu"` Mem NodeStatsOSMemResponse `json:"mem"` Swap NodeStatsOSSwapResponse `json:"swap"` } // NodeStatsOSMemResponse defines node stats operating system memory usage structure type NodeStatsOSMemResponse struct { Free int64 `json:"free_in_bytes"` Used int64 `json:"used_in_bytes"` ActualFree int64 `json:"actual_free_in_bytes"` ActualUsed int64 `json:"actual_used_in_bytes"` } // NodeStatsOSSwapResponse defines node stats operating system swap usage structure type NodeStatsOSSwapResponse struct { Used int64 `json:"used_in_bytes"` Free int64 `json:"free_in_bytes"` } // NodeStatsOSCPUResponse defines node stats operating system CPU usage structure type NodeStatsOSCPUResponse struct { Sys int64 `json:"sys"` User int64 `json:"user"` Idle int64 `json:"idle"` Steal int64 `json:"stolen"` LoadAvg NodeStatsOSCPULoadResponse `json:"load_average"` Percent int64 `json:"percent"` } // NodeStatsOSCPULoadResponse defines node stats operating system CPU load structure type NodeStatsOSCPULoadResponse struct { Load1 float64 `json:"1m"` Load5 float64 `json:"5m"` Load15 float64 `json:"15m"` } // NodeStatsProcessResponse is a representation of a process statistics, memory consumption, cpu usage, open file descriptors type NodeStatsProcessResponse struct { Timestamp int64 `json:"timestamp"` OpenFD int64 `json:"open_file_descriptors"` MaxFD int64 `json:"max_file_descriptors"` CPU NodeStatsProcessCPUResponse `json:"cpu"` Memory NodeStatsProcessMemResponse `json:"mem"` } // NodeStatsProcessMemResponse defines node stats process memory usage structure type NodeStatsProcessMemResponse struct { Resident int64 `json:"resident_in_bytes"` Share int64 `json:"share_in_bytes"` TotalVirtual int64 `json:"total_virtual_in_bytes"` } // NodeStatsProcessCPUResponse defines node stats process CPU usage structure type NodeStatsProcessCPUResponse struct { Percent int64 `json:"percent"` Sys int64 `json:"sys_in_millis"` User int64 `json:"user_in_millis"` Total int64 `json:"total_in_millis"` } // NodeStatsHTTPResponse defines node stats HTTP connections structure type NodeStatsHTTPResponse struct { CurrentOpen int64 `json:"current_open"` TotalOpen int64 `json:"total_open"` } // NodeStatsFSResponse is a representation of a file system information, data path, free disk space, read/write stats type NodeStatsFSResponse struct { Timestamp int64 `json:"timestamp"` Data []NodeStatsFSDataResponse `json:"data"` IOStats NodeStatsFSIOStatsResponse `json:"io_stats"` } // NodeStatsFSDataResponse defines node stats filesystem data structure type NodeStatsFSDataResponse struct { Path string `json:"path"` Mount string `json:"mount"` Device string `json:"dev"` Total int64 `json:"total_in_bytes"` Free int64 `json:"free_in_bytes"` Available int64 `json:"available_in_bytes"` } // NodeStatsFSIOStatsResponse defines node stats filesystem device structure type NodeStatsFSIOStatsResponse struct { Devices []NodeStatsFSIOStatsDeviceResponse `json:"devices"` } // NodeStatsFSIOStatsDeviceResponse is a representation of a node stat filesystem device type NodeStatsFSIOStatsDeviceResponse struct { DeviceName string `json:"device_name"` Operations int64 `json:"operations"` ReadOperations int64 `json:"read_operations"` WriteOperations int64 `json:"write_operations"` ReadSize int64 `json:"read_kilobytes"` WriteSize int64 `json:"write_kilobytes"` } // ClusterHealthResponse is a representation of a Elasticsearch Cluster Health type ClusterHealthResponse struct { ActivePrimaryShards int64 `json:"active_primary_shards"` ActiveShards int64 `json:"active_shards"` ClusterName string `json:"cluster_name"` DelayedUnassignedShards int64 `json:"delayed_unassigned_shards"` InitializingShards int64 `json:"initializing_shards"` NumberOfDataNodes int64 `json:"number_of_data_nodes"` NumberOfInFlightFetch int64 `json:"number_of_in_flight_fetch"` NumberOfNodes int64 `json:"number_of_nodes"` NumberOfPendingTasks int64 `json:"number_of_pending_tasks"` RelocatingShards int64 `json:"relocating_shards"` Status string `json:"status"` TimedOut bool `json:"timed_out"` UnassignedShards int64 `json:"unassigned_shards"` } prometheus-elasticsearch-exporter-1.1.0+ds/collector/nodes_test.go000066400000000000000000003502261362377146000254710ustar00rootroot00000000000000package collector import ( "encoding/base64" "fmt" "net/http" "net/http/httptest" "net/url" "strings" "testing" "github.com/go-kit/kit/log" ) func TestNodesStats(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine // curl -XPUT http://localhost:9200/twitter // curl http://localhost:9200/_nodes/stats tcs := map[string]string{ "1.7.6": `{"cluster_name":"elasticsearch","nodes":{"vKnRv2bPThCJIDTj92NvQg":{"timestamp":1498820641883,"name":"Overmind","transport_address":"inet[/172.17.0.4:9300]","host":"2603876b583f","ip":["inet[/172.17.0.4:9300]","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":575,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":52},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820641896},"process":{"timestamp":1498820641896,"open_file_descriptors":123},"jvm":{"timestamp":1498820641896,"uptime_in_millis":17871,"mem":{"heap_used_in_bytes":82958528,"heap_used_percent":7,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":44442536,"non_heap_committed_in_bytes":45400064,"pools":{"young":{"used_in_bytes":62150064,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":3375888,"max_in_bytes":899284992,"peak_used_in_bytes":10899920,"peak_max_in_bytes":899284992}}},"threads":{"count":36,"peak_count":36},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":40},"old":{"collection_count":1,"collection_time_in_millis":42}}},"buffer_pools":{"direct":{"count":27,"used_in_bytes":4981411,"total_capacity_in_bytes":4981411},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":14},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":1},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"network":{},"fs":{"timestamp":1498820641898,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1428,"tx_count":6,"tx_size_in_bytes":1428},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}}}}}`, "2.4.5": `{"cluster_name":"elasticsearch","nodes":{"VnnrklnAQ7KRXhW2WkgPxA":{"timestamp":1498820602662,"name":"Battering Ram","transport_address":"172.17.0.4:9300","host":"172.17.0.4","ip":["172.17.0.4:9300","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":60},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":105630920,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":215},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820602670,"cpu_percent":7,"load_average":0.32861328125,"mem":{"total_in_bytes":2096177152,"free_in_bytes":1624637440,"used_in_bytes":471539712,"free_percent":78,"used_percent":22},"swap":{"total_in_bytes":4195348480,"free_in_bytes":4005060608,"used_in_bytes":190287872}},"process":{"timestamp":1498820602670,"open_file_descriptors":116,"max_file_descriptors":1048576,"cpu":{"percent":7,"total_in_millis":10270},"mem":{"total_virtual_in_bytes":2681352192}},"jvm":{"timestamp":1498820602671,"uptime_in_millis":38833,"mem":{"heap_used_in_bytes":88986784,"heap_used_percent":8,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":49185912,"non_heap_committed_in_bytes":49987584,"pools":{"young":{"used_in_bytes":58281984,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":13272224,"max_in_bytes":899284992,"peak_used_in_bytes":13272224,"peak_max_in_bytes":899284992}}},"threads":{"count":35,"peak_count":38},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":87},"old":{"collection_count":1,"collection_time_in_millis":74}}},"buffer_pools":{"direct":{"count":23,"used_in_bytes":3416836,"total_capacity_in_bytes":3416836},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":7221,"total_loaded_count":7221,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":5,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":2},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":5},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1498820602672,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"}]},"transport":{"server_open":0,"rx_count":6,"rx_size_in_bytes":2028,"tx_count":6,"tx_size_in_bytes":2028},"http":{"current_open":1,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":1056309248,"limit_size":"1007.3mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0}}}}`, "5.4.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"0hHcEFK1S7qMlk8hQCm7wQ":{"timestamp":1498820489394,"name":"0hHcEFK","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":27},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":215},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820489400,"cpu":{"percent":0,"load_average":{"1m":0.35,"5m":0.28,"15m":0.12}},"mem":{"total_in_bytes":2096177152,"free_in_bytes":83501056,"used_in_bytes":2012676096,"free_percent":4,"used_percent":96},"swap":{"total_in_bytes":4195348480,"free_in_bytes":3487707136,"used_in_bytes":707641344}},"process":{"timestamp":1498820489400,"open_file_descriptors":139,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":26600},"mem":{"total_virtual_in_bytes":3823181824}},"jvm":{"timestamp":1498820489400,"uptime_in_millis":185693,"mem":{"heap_used_in_bytes":114959064,"heap_used_percent":5,"heap_committed_in_bytes":2130051072,"heap_max_in_bytes":2130051072,"non_heap_used_in_bytes":65471864,"non_heap_committed_in_bytes":69906432,"pools":{"young":{"used_in_bytes":82057312,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":15469176,"max_in_bytes":1973026816,"peak_used_in_bytes":15469176,"peak_max_in_bytes":1973026816}}},"threads":{"count":26,"peak_count":29},"gc":{"collectors":{"young":{"collection_count":4,"collection_time_in_millis":1618},"old":{"collection_count":1,"collection_time_in_millis":76}}},"buffer_pools":{"direct":{"count":16,"used_in_bytes":33776600,"total_capacity_in_bytes":33776599},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":9995,"total_loaded_count":9995,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":20},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1498820489401,"total":{"total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sda2","operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}],"total":{"operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":2,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2130051072,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1491035750,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}`, "5.5.0": `{"_nodes":{"total":13,"successful":13,"failed":0},"cluster_name":"multinode","nodes":{"IAMD9JnLR4qOa-dl3cDaig":{"timestamp":1511714498349,"name":"elasticin-02","transport_address":"10.0.0.108:9300","host":"elasticin-02","ip":"10.0.0.108:9300","roles":["ingest"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714497950,"cpu":{"percent":1,"load_average":{"1m":0.06,"5m":0.21,"15m":0.29}},"mem":{"total_in_bytes":8375726080,"free_in_bytes":545091584,"used_in_bytes":7830634496,"free_percent":7,"used_percent":93},"swap":{"total_in_bytes":1073737728,"free_in_bytes":1073737728,"used_in_bytes":0},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":37998635799304},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497950,"open_file_descriptors":507,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":37998520},"mem":{"total_virtual_in_bytes":8281767936}},"jvm":{"timestamp":1511714497950,"uptime_in_millis":589198476,"mem":{"heap_used_in_bytes":1092031200,"heap_used_percent":25,"heap_committed_in_bytes":4260102144,"heap_max_in_bytes":4260102144,"non_heap_used_in_bytes":114726896,"non_heap_committed_in_bytes":121688064,"pools":{"young":{"used_in_bytes":158592080,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":2988992,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":930450128,"max_in_bytes":3946053632,"peak_used_in_bytes":3806199632,"peak_max_in_bytes":3946053632}}},"threads":{"count":60,"peak_count":185},"gc":{"collectors":{"young":{"collection_count":60237,"collection_time_in_millis":1196377},"old":{"collection_count":287,"collection_time_in_millis":38202}}},"buffer_pools":{"direct":{"count":40,"used_in_bytes":135831560,"total_capacity_in_bytes":135831559},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":13297,"total_loaded_count":13364,"total_unloaded_count":67}},"thread_pool":{"bulk":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":404400},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":128,"completed":765207},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":1289143},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497951,"total":{"total_in_bytes":39836917760,"free_in_bytes":35574710272,"available_in_bytes":33527472128,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":35574710272,"available_in_bytes":33527472128,"spins":"true"}],"io_stats":{}},"transport":{"server_open":135,"rx_count":5047176,"rx_size_in_bytes":154154850912,"tx_count":5049542,"tx_size_in_bytes":228013049059},"http":{"current_open":2,"total_opened":581038},"breakers":{"request":{"limit_size_in_bytes":2556061286,"limit_size":"2.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":2556061286,"limit_size":"2.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":4260102144,"limit_size":"3.9gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":2982071500,"limit_size":"2.7gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"o9MvLOdySpmaelLxYnLJDg":{"timestamp":1511714498348,"name":"elasticmaster-01","transport_address":"10.0.0.150:9300","host":"elasticmaster-01","ip":"10.0.0.150:9300","roles":["master"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714498349,"cpu":{"percent":44,"load_average":{"1m":0.25,"5m":0.33,"15m":0.25}},"mem":{"total_in_bytes":8375730176,"free_in_bytes":2545594368,"used_in_bytes":5830135808,"free_percent":30,"used_percent":70},"swap":{"total_in_bytes":1073737728,"free_in_bytes":1072058368,"used_in_bytes":1679360},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":20472909415977},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714498349,"open_file_descriptors":519,"max_file_descriptors":1048576,"cpu":{"percent":2,"total_in_millis":20472800},"mem":{"total_virtual_in_bytes":6048481280}},"jvm":{"timestamp":1511714498349,"uptime_in_millis":455012435,"mem":{"heap_used_in_bytes":1196945872,"heap_used_percent":56,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":113206520,"non_heap_committed_in_bytes":120074240,"pools":{"young":{"used_in_bytes":109972848,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":2172240,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":1084800784,"max_in_bytes":1798569984,"peak_used_in_bytes":1798569984,"peak_max_in_bytes":1798569984}}},"threads":{"count":52,"peak_count":56},"gc":{"collectors":{"young":{"collection_count":32035,"collection_time_in_millis":355769},"old":{"collection_count":1002,"collection_time_in_millis":476417}}},"buffer_pools":{"direct":{"count":41,"used_in_bytes":135839401,"total_capacity_in_bytes":135839400},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":13299,"total_loaded_count":13359,"total_unloaded_count":60}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":590358},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":949237},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714498349,"total":{"total_in_bytes":39836917760,"free_in_bytes":27525136384,"available_in_bytes":25477898240,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":27525136384,"available_in_bytes":25477898240,"spins":"true"}],"io_stats":{}},"transport":{"server_open":134,"rx_count":3128979,"rx_size_in_bytes":44609852501,"tx_count":3128980,"tx_size_in_bytes":91308571728},"http":{"current_open":2,"total_opened":407857},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"QJMwXyZDTVaEupifVTFAXg":{"timestamp":1511714498348,"name":"elasticmaster-03","transport_address":"10.0.0.57:9300","host":"elasticmaster-03","ip":"10.0.0.57:9300","roles":["master"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714497945,"cpu":{"percent":23,"load_average":{"1m":0.81,"5m":0.67,"15m":0.53}},"mem":{"total_in_bytes":8375726080,"free_in_bytes":2463612928,"used_in_bytes":5912113152,"free_percent":29,"used_percent":71},"swap":{"total_in_bytes":1073737728,"free_in_bytes":1071788032,"used_in_bytes":1949696},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":72007272720599},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":519,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":72007150},"mem":{"total_virtual_in_bytes":6073102336}},"jvm":{"timestamp":1511714497945,"uptime_in_millis":456828928,"mem":{"heap_used_in_bytes":830571920,"heap_used_percent":39,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":131650000,"non_heap_committed_in_bytes":140623872,"pools":{"young":{"used_in_bytes":42273680,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":8297248,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":780000992,"max_in_bytes":1798569984,"peak_used_in_bytes":1798569984,"peak_max_in_bytes":1798569984}}},"threads":{"count":74,"peak_count":75},"gc":{"collectors":{"young":{"collection_count":187631,"collection_time_in_millis":2346463},"old":{"collection_count":1570,"collection_time_in_millis":494403}}},"buffer_pools":{"direct":{"count":41,"used_in_bytes":135839383,"total_capacity_in_bytes":135839382},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":14015,"total_loaded_count":14100,"total_unloaded_count":85}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":18,"queue":0,"active":1,"rejected":0,"largest":18,"completed":592984},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":1026939},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":5,"queue":0,"active":0,"rejected":0,"largest":5,"completed":5},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":2,"queue":0,"active":0,"rejected":0,"largest":2,"completed":2},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497945,"total":{"total_in_bytes":39836917760,"free_in_bytes":31222030336,"available_in_bytes":29174792192,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":31222030336,"available_in_bytes":29174792192,"spins":"true"}],"io_stats":{}},"transport":{"server_open":134,"rx_count":17417277,"rx_size_in_bytes":388077081117,"tx_count":17417313,"tx_size_in_bytes":320472005934},"http":{"current_open":2,"total_opened":407233},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"EN05ytfQQvStXdW4Q6RatA":{"timestamp":1511714498348,"name":"elasticdata-01","transport_address":"10.0.0.119:9300","host":"elasticdata-01","ip":"10.0.0.119:9300","roles":["data"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":375322138,"deleted":1037878},"store":{"size_in_bytes":259394317595,"throttle_time_in_millis":0},"indexing":{"index_total":448977978,"index_time_in_millis":118716971,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":11762,"time_in_millis":941,"exists_total":11762,"exists_time_in_millis":941,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":482173,"query_time_in_millis":3748653,"query_current":0,"fetch_total":21577,"fetch_time_in_millis":528409,"fetch_current":0,"scroll_total":2,"scroll_time_in_millis":188893,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":2,"current_docs":9550,"current_size_in_bytes":6050542,"total":169074,"total_time_in_millis":105409372,"total_docs":1390928712,"total_size_in_bytes":869934290826,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":50576302,"total_auto_throttle_in_bytes":14087997090},"refresh":{"total":1476283,"total_time_in_millis":27373898,"listeners":0},"flush":{"total":1184,"total_time_in_millis":518446},"warmer":{"current":0,"total":1441617,"total_time_in_millis":520582},"query_cache":{"memory_size_in_bytes":154620318,"total_count":1276959,"hit_count":655594,"miss_count":621365,"cache_size":8688,"cache_count":11271,"evictions":2583},"fielddata":{"memory_size_in_bytes":50488,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4460,"memory_in_bytes":906650903,"terms_memory_in_bytes":814497094,"stored_fields_memory_in_bytes":62547264,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":8536576,"points_memory_in_bytes":4277057,"doc_values_memory_in_bytes":16792912,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":192,"max_unsafe_auto_id_timestamp":1511481600647,"file_sizes":{}},"translog":{"operations":3209732,"size_in_bytes":3144619388},"request_cache":{"memory_size_in_bytes":162916154,"evictions":12109,"hit_count":19313,"miss_count":37864},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":743800}},"os":{"timestamp":1511714497945,"cpu":{"percent":5,"load_average":{"1m":0.98,"5m":0.92,"15m":0.81}},"mem":{"total_in_bytes":33742192640,"free_in_bytes":1784082432,"used_in_bytes":31958110208,"free_percent":5,"used_percent":95},"swap":{"total_in_bytes":1073737728,"free_in_bytes":791388160,"used_in_bytes":282349568},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":377534011797418},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":1341,"max_file_descriptors":1048576,"cpu":{"percent":4,"total_in_millis":377531950},"mem":{"total_virtual_in_bytes":281489690624}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":456177136,"mem":{"heap_used_in_bytes":12360895776,"heap_used_percent":72,"heap_committed_in_bytes":17110138880,"heap_max_in_bytes":17110138880,"non_heap_used_in_bytes":169873944,"non_heap_committed_in_bytes":179277824,"pools":{"young":{"used_in_bytes":418031976,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":41567680,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":11901296120,"max_in_bytes":16481976320,"peak_used_in_bytes":12805122832,"peak_max_in_bytes":16481976320}}},"threads":{"count":134,"peak_count":141},"gc":{"collectors":{"young":{"collection_count":138863,"collection_time_in_millis":4901025},"old":{"collection_count":109,"collection_time_in_millis":28223}}},"buffer_pools":{"direct":{"count":153,"used_in_bytes":272389833,"total_capacity_in_bytes":272389832},"mapped":{"count":9987,"used_in_bytes":257693104518,"total_capacity_in_bytes":257693104518}},"classes":{"current_loaded_count":15279,"total_loaded_count":15425,"total_unloaded_count":146}},"thread_pool":{"bulk":{"threads":8,"queue":0,"active":0,"rejected":8829,"largest":8,"completed":4533781},"fetch_shard_started":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"fetch_shard_store":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":527},"flush":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":183951},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":16,"queue":0,"active":0,"rejected":0,"largest":16,"completed":685858},"get":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":11757},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":2,"rejected":0,"largest":5,"completed":1482328},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":96747907},"search":{"threads":13,"queue":0,"active":0,"rejected":0,"largest":13,"completed":492085},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":3,"queue":0,"active":0,"rejected":0,"largest":3,"completed":1478002},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":528311836672,"free_in_bytes":259901870080,"available_in_bytes":233041600512,"spins":"true"},"data":[{"path":"/mnt/data/nodes/0","mount":"/mnt/data (/dev/sdb1)","type":"ext4","total_in_bytes":528311836672,"free_in_bytes":259901870080,"available_in_bytes":233041600512,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sdb1","operations":21613740,"read_operations":757427,"write_operations":20856313,"read_kilobytes":82400388,"write_kilobytes":1355771052}],"total":{"operations":21613740,"read_operations":757427,"write_operations":20856313,"read_kilobytes":82400388,"write_kilobytes":1355771052}}},"transport":{"server_open":133,"rx_count":11869543,"rx_size_in_bytes":477850961639,"tx_count":11869509,"tx_size_in_bytes":538588051057},"http":{"current_open":2,"total_opened":396888},"breakers":{"request":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":50488,"estimated_size":"49.3kb","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":17110138880,"limit_size":"15.9gb","estimated_size_in_bytes":38028,"estimated_size":"37.1kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":11977097216,"limit_size":"11.1gb","estimated_size_in_bytes":88516,"estimated_size":"86.4kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"CC7h3_tGQTm7sUk0sMAPjg":{"timestamp":1511714498349,"name":"elasticmaster-02","transport_address":"10.0.0.149:9300","host":"elasticmaster-02","ip":"10.0.0.149:9300","roles":["master"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714497946,"cpu":{"percent":11,"load_average":{"1m":0.04,"5m":0.03,"15m":0.05}},"mem":{"total_in_bytes":8375726080,"free_in_bytes":2514419712,"used_in_bytes":5861306368,"free_percent":30,"used_percent":70},"swap":{"total_in_bytes":1073737728,"free_in_bytes":1070624768,"used_in_bytes":3112960},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":15116859468711},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497946,"open_file_descriptors":519,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":15116720},"mem":{"total_virtual_in_bytes":6050123776}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":439476155,"mem":{"heap_used_in_bytes":723824488,"heap_used_percent":34,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":111904440,"non_heap_committed_in_bytes":119291904,"pools":{"young":{"used_in_bytes":215917352,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":6213480,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":501693656,"max_in_bytes":1798569984,"peak_used_in_bytes":1669293512,"peak_max_in_bytes":1798569984}}},"threads":{"count":52,"peak_count":57},"gc":{"collectors":{"young":{"collection_count":21247,"collection_time_in_millis":216235},"old":{"collection_count":155,"collection_time_in_millis":16680}}},"buffer_pools":{"direct":{"count":41,"used_in_bytes":135839407,"total_capacity_in_bytes":135839406},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":13220,"total_loaded_count":13275,"total_unloaded_count":55}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":570794},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":896859},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":39836917760,"free_in_bytes":24176267264,"available_in_bytes":22129029120,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":24176267264,"available_in_bytes":22129029120,"spins":"true"}],"io_stats":{}},"transport":{"server_open":134,"rx_count":2973754,"rx_size_in_bytes":41620665024,"tx_count":2973754,"tx_size_in_bytes":68323082688},"http":{"current_open":2,"total_opened":389190},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"Pc2mK8pWQ9icfpJRpEqLWw":{"timestamp":1511714498348,"name":"elasticdata-04","transport_address":"10.0.0.56:9300","host":"elasticdata-04","ip":"10.0.0.56:9300","roles":["data"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":425699379,"deleted":1373715},"store":{"size_in_bytes":296033664114,"throttle_time_in_millis":0},"indexing":{"index_total":642934027,"index_time_in_millis":171016063,"index_current":0,"index_failed":35069830,"delete_total":1721,"delete_time_in_millis":132,"delete_current":0,"noop_update_total":2,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":15655,"time_in_millis":905,"exists_total":15647,"exists_time_in_millis":905,"missing_total":8,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":650749,"query_time_in_millis":3957392,"query_current":0,"fetch_total":21133,"fetch_time_in_millis":611629,"fetch_current":0,"scroll_total":49,"scroll_time_in_millis":2195819,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":208750,"total_time_in_millis":151039152,"total_docs":1916131698,"total_size_in_bytes":1226965745190,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":83417760,"total_auto_throttle_in_bytes":41028302003},"refresh":{"total":1825950,"total_time_in_millis":35474740,"listeners":0},"flush":{"total":3138,"total_time_in_millis":707216},"warmer":{"current":0,"total":1453839,"total_time_in_millis":657162},"query_cache":{"memory_size_in_bytes":87457278,"total_count":479101,"hit_count":109644,"miss_count":369457,"cache_size":15020,"cache_count":15792,"evictions":772},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4558,"memory_in_bytes":1094352149,"terms_memory_in_bytes":997400697,"stored_fields_memory_in_bytes":72061816,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":8919296,"points_memory_in_bytes":2447676,"doc_values_memory_in_bytes":13522664,"index_writer_memory_in_bytes":2321128,"version_map_memory_in_bytes":10230,"fixed_bit_set_memory_in_bytes":96,"max_unsafe_auto_id_timestamp":1511222737106,"file_sizes":{}},"translog":{"operations":4076380,"size_in_bytes":3763408908},"request_cache":{"memory_size_in_bytes":522081,"evictions":0,"hit_count":1057,"miss_count":715},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":6095665}},"os":{"timestamp":1511714497945,"cpu":{"percent":8,"load_average":{"1m":0.49,"5m":0.75,"15m":0.83}},"mem":{"total_in_bytes":33742192640,"free_in_bytes":326279168,"used_in_bytes":33415913472,"free_percent":1,"used_percent":99},"swap":{"total_in_bytes":1073737728,"free_in_bytes":734015488,"used_in_bytes":339722240},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":505471512208338},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":1495,"max_file_descriptors":1048576,"cpu":{"percent":8,"total_in_millis":505469110},"mem":{"total_virtual_in_bytes":317985972224}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":584091461,"mem":{"heap_used_in_bytes":8628353304,"heap_used_percent":50,"heap_committed_in_bytes":17110138880,"heap_max_in_bytes":17110138880,"non_heap_used_in_bytes":182671192,"non_heap_committed_in_bytes":192577536,"pools":{"young":{"used_in_bytes":318839256,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":25041808,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":8284472240,"max_in_bytes":16481976320,"peak_used_in_bytes":16481976320,"peak_max_in_bytes":16481976320}}},"threads":{"count":140,"peak_count":220},"gc":{"collectors":{"young":{"collection_count":198328,"collection_time_in_millis":7115355},"old":{"collection_count":229,"collection_time_in_millis":602402}}},"buffer_pools":{"direct":{"count":143,"used_in_bytes":272254590,"total_capacity_in_bytes":272254589},"mapped":{"count":10120,"used_in_bytes":293958375802,"total_capacity_in_bytes":293958375802}},"classes":{"current_loaded_count":15336,"total_loaded_count":15495,"total_unloaded_count":159}},"thread_pool":{"bulk":{"threads":8,"queue":0,"active":0,"rejected":106834,"largest":8,"completed":6643785},"fetch_shard_started":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":2854},"fetch_shard_store":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":10424},"flush":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":234813},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":12,"queue":0,"active":0,"rejected":0,"largest":60,"completed":1065873},"get":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":15629},"index":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":12},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":2,"rejected":0,"largest":5,"completed":1911527},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":124094775},"search":{"threads":13,"queue":0,"active":0,"rejected":0,"largest":13,"completed":657456},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":1831272},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":528310788096,"free_in_bytes":219653083136,"available_in_bytes":219636305920,"spins":"true"},"data":[{"path":"/mnt/data/nodes/0","mount":"/mnt/data (/dev/sdb1)","type":"ext4","total_in_bytes":528310788096,"free_in_bytes":219653083136,"available_in_bytes":219636305920,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sdb1","operations":34428907,"read_operations":4433749,"write_operations":29995158,"read_kilobytes":530811024,"write_kilobytes":2036811840}],"total":{"operations":34428907,"read_operations":4433749,"write_operations":29995158,"read_kilobytes":530811024,"write_kilobytes":2036811840}}},"transport":{"server_open":133,"rx_count":16198439,"rx_size_in_bytes":900601319307,"tx_count":16201772,"tx_size_in_bytes":1162809703365},"http":{"current_open":2,"total_opened":498058},"breakers":{"request":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":17110138880,"limit_size":"15.9gb","estimated_size_in_bytes":38013,"estimated_size":"37.1kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":11977097216,"limit_size":"11.1gb","estimated_size_in_bytes":38013,"estimated_size":"37.1kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"vQxBfSGXTuyFRZ0qVrNrDA":{"timestamp":1511714498348,"name":"elasticdata-05","transport_address":"10.0.0.98:9300","host":"elasticdata-05","ip":"10.0.0.98:9300","roles":["data"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":464562958,"deleted":1135485},"store":{"size_in_bytes":293883305118,"throttle_time_in_millis":0},"indexing":{"index_total":332603493,"index_time_in_millis":105361198,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":166531,"time_in_millis":13112,"exists_total":166531,"exists_time_in_millis":13112,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":654756,"query_time_in_millis":3925477,"query_current":0,"fetch_total":168250,"fetch_time_in_millis":168976,"fetch_current":0,"scroll_total":1,"scroll_time_in_millis":93623,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":166059,"total_time_in_millis":126117143,"total_docs":1332039112,"total_size_in_bytes":927972166158,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":71787253,"total_auto_throttle_in_bytes":9894986790},"refresh":{"total":1435483,"total_time_in_millis":25368823,"listeners":0},"flush":{"total":966,"total_time_in_millis":304725},"warmer":{"current":0,"total":1429821,"total_time_in_millis":655606},"query_cache":{"memory_size_in_bytes":58846620,"total_count":962195,"hit_count":493138,"miss_count":469057,"cache_size":15951,"cache_count":16868,"evictions":917},"fielddata":{"memory_size_in_bytes":50008,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4349,"memory_in_bytes":1064790107,"terms_memory_in_bytes":968028700,"stored_fields_memory_in_bytes":69207552,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":8031040,"points_memory_in_bytes":4115219,"doc_values_memory_in_bytes":15407596,"index_writer_memory_in_bytes":2463588,"version_map_memory_in_bytes":9020,"fixed_bit_set_memory_in_bytes":680,"max_unsafe_auto_id_timestamp":1511291784471,"file_sizes":{}},"translog":{"operations":3779808,"size_in_bytes":3315625571},"request_cache":{"memory_size_in_bytes":163650616,"evictions":0,"hit_count":24539,"miss_count":33213},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":2115817}},"os":{"timestamp":1511714497945,"cpu":{"percent":9,"load_average":{"1m":0.78,"5m":0.81,"15m":0.96}},"mem":{"total_in_bytes":33742192640,"free_in_bytes":398262272,"used_in_bytes":33343930368,"free_percent":1,"used_percent":99},"swap":{"total_in_bytes":1073737728,"free_in_bytes":759992320,"used_in_bytes":313745408},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":354202797708211},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":1342,"max_file_descriptors":1048576,"cpu":{"percent":6,"total_in_millis":354200650},"mem":{"total_virtual_in_bytes":315156033536}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":431765693,"mem":{"heap_used_in_bytes":6319079512,"heap_used_percent":36,"heap_committed_in_bytes":17110138880,"heap_max_in_bytes":17110138880,"non_heap_used_in_bytes":170115336,"non_heap_committed_in_bytes":178569216,"pools":{"young":{"used_in_bytes":443763064,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":14138776,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":5861177672,"max_in_bytes":16481976320,"peak_used_in_bytes":12466824144,"peak_max_in_bytes":16481976320}}},"threads":{"count":123,"peak_count":129},"gc":{"collectors":{"young":{"collection_count":120635,"collection_time_in_millis":4020956},"old":{"collection_count":34,"collection_time_in_millis":7882}}},"buffer_pools":{"direct":{"count":121,"used_in_bytes":272138248,"total_capacity_in_bytes":272138247},"mapped":{"count":9931,"used_in_bytes":291373910869,"total_capacity_in_bytes":291373910869}},"classes":{"current_loaded_count":15157,"total_loaded_count":15278,"total_unloaded_count":121}},"thread_pool":{"bulk":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":4880397},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":172938},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":6,"queue":0,"active":0,"rejected":0,"largest":6,"completed":988384},"get":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":166525},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":4,"rejected":0,"largest":5,"completed":1392916},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":95595890},"search":{"threads":13,"queue":0,"active":0,"rejected":0,"largest":13,"completed":656498},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":1436871},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":528310788096,"free_in_bytes":230955548672,"available_in_bytes":230938771456,"spins":"true"},"data":[{"path":"/mnt/data/nodes/0","mount":"/mnt/data (/dev/sdb1)","type":"ext4","total_in_bytes":528310788096,"free_in_bytes":230955548672,"available_in_bytes":230938771456,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sdb1","operations":23957106,"read_operations":1813100,"write_operations":22144006,"read_kilobytes":208071952,"write_kilobytes":1567856356}],"total":{"operations":23957106,"read_operations":1813100,"write_operations":22144006,"read_kilobytes":208071952,"write_kilobytes":1567856356}}},"transport":{"server_open":133,"rx_count":11052974,"rx_size_in_bytes":548594647188,"tx_count":11052976,"tx_size_in_bytes":267372947055},"http":{"current_open":3,"total_opened":383162},"breakers":{"request":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":50008,"estimated_size":"48.8kb","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":17110138880,"limit_size":"15.9gb","estimated_size_in_bytes":73996,"estimated_size":"72.2kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":11977097216,"limit_size":"11.1gb","estimated_size_in_bytes":124004,"estimated_size":"121kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"sJu6S4ZOSNKj2GewgLAA6w":{"timestamp":1511714498348,"name":"elasticin-01","transport_address":"10.0.0.109:9300","host":"elasticin-01","ip":"10.0.0.109:9300","roles":["ingest"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714497945,"cpu":{"percent":5,"load_average":{"1m":0.33,"5m":0.37,"15m":0.32}},"mem":{"total_in_bytes":8375726080,"free_in_bytes":518225920,"used_in_bytes":7857500160,"free_percent":6,"used_percent":94},"swap":{"total_in_bytes":1073737728,"free_in_bytes":1073737728,"used_in_bytes":0},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":36799094034870},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":507,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":36798970},"mem":{"total_virtual_in_bytes":8281767936}},"jvm":{"timestamp":1511714497945,"uptime_in_millis":589205044,"mem":{"heap_used_in_bytes":1052113704,"heap_used_percent":24,"heap_committed_in_bytes":4260102144,"heap_max_in_bytes":4260102144,"non_heap_used_in_bytes":115304704,"non_heap_committed_in_bytes":122564608,"pools":{"young":{"used_in_bytes":49586432,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":3992336,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":998591504,"max_in_bytes":3946053632,"peak_used_in_bytes":3585864320,"peak_max_in_bytes":3946053632}}},"threads":{"count":59,"peak_count":161},"gc":{"collectors":{"young":{"collection_count":57750,"collection_time_in_millis":1081864},"old":{"collection_count":257,"collection_time_in_millis":23972}}},"buffer_pools":{"direct":{"count":40,"used_in_bytes":135831560,"total_capacity_in_bytes":135831559},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":13399,"total_loaded_count":13478,"total_unloaded_count":79}},"thread_pool":{"bulk":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":404291},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":7,"queue":0,"active":0,"rejected":0,"largest":109,"completed":765856},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":1288944},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497945,"total":{"total_in_bytes":39836917760,"free_in_bytes":35580149760,"available_in_bytes":33532911616,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":35580149760,"available_in_bytes":33532911616,"spins":"true"}],"io_stats":{}},"transport":{"server_open":135,"rx_count":5045776,"rx_size_in_bytes":152482073745,"tx_count":5048384,"tx_size_in_bytes":224365671464},"http":{"current_open":2,"total_opened":580259},"breakers":{"request":{"limit_size_in_bytes":2556061286,"limit_size":"2.3gb","estimated_size_in_bytes":230160,"estimated_size":"224.7kb","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":2556061286,"limit_size":"2.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":4260102144,"limit_size":"3.9gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":2982071500,"limit_size":"2.7gb","estimated_size_in_bytes":232076,"estimated_size":"226.6kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"bGzxfiCWT5KcR7jF4ADJ_g":{"timestamp":1511714498349,"name":"elasticdata-03","transport_address":"10.0.0.55:9300","host":"elasticdata-03","ip":"10.0.0.55:9300","roles":["data"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":501080121,"deleted":1190750},"store":{"size_in_bytes":315723703383,"throttle_time_in_millis":0},"indexing":{"index_total":650070635,"index_time_in_millis":167204215,"index_current":0,"index_failed":1332739,"delete_total":1968,"delete_time_in_millis":194,"delete_current":0,"noop_update_total":133,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":30909,"time_in_millis":2520,"exists_total":30907,"exists_time_in_millis":2520,"missing_total":2,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":639532,"query_time_in_millis":4571412,"query_current":0,"fetch_total":37389,"fetch_time_in_millis":510596,"fetch_current":0,"scroll_total":38,"scroll_time_in_millis":1484669,"scroll_current":1,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":207892,"total_time_in_millis":156215892,"total_docs":1938519154,"total_size_in_bytes":1192900048810,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":85295109,"total_auto_throttle_in_bytes":56331279192},"refresh":{"total":1794344,"total_time_in_millis":35971432,"listeners":0},"flush":{"total":3365,"total_time_in_millis":678944},"warmer":{"current":0,"total":1484801,"total_time_in_millis":689654},"query_cache":{"memory_size_in_bytes":111403405,"total_count":556164,"hit_count":117910,"miss_count":438254,"cache_size":17962,"cache_count":19094,"evictions":1132},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4518,"memory_in_bytes":1141279472,"terms_memory_in_bytes":1037548187,"stored_fields_memory_in_bytes":77990264,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":8918784,"points_memory_in_bytes":2803229,"doc_values_memory_in_bytes":14019008,"index_writer_memory_in_bytes":3135060,"version_map_memory_in_bytes":4650,"fixed_bit_set_memory_in_bytes":96,"max_unsafe_auto_id_timestamp":1511222421028,"file_sizes":{}},"translog":{"operations":3730929,"size_in_bytes":3098662483},"request_cache":{"memory_size_in_bytes":246594,"evictions":0,"hit_count":497,"miss_count":396},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":7727317}},"os":{"timestamp":1511714497945,"cpu":{"percent":7,"load_average":{"1m":0.57,"5m":0.47,"15m":0.66}},"mem":{"total_in_bytes":33742192640,"free_in_bytes":307798016,"used_in_bytes":33434394624,"free_percent":1,"used_percent":99},"swap":{"total_in_bytes":1073737728,"free_in_bytes":725217280,"used_in_bytes":348520448},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":511903359994882},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":1518,"max_file_descriptors":1048576,"cpu":{"percent":5,"total_in_millis":511900740},"mem":{"total_virtual_in_bytes":338006835200}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":584100083,"mem":{"heap_used_in_bytes":4294670120,"heap_used_percent":25,"heap_committed_in_bytes":17110138880,"heap_max_in_bytes":17110138880,"non_heap_used_in_bytes":179614824,"non_heap_committed_in_bytes":189874176,"pools":{"young":{"used_in_bytes":236457480,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":22430736,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":4035781904,"max_in_bytes":16481976320,"peak_used_in_bytes":13487381760,"peak_max_in_bytes":16481976320}}},"threads":{"count":136,"peak_count":273},"gc":{"collectors":{"young":{"collection_count":193711,"collection_time_in_millis":6942549},"old":{"collection_count":255,"collection_time_in_millis":56702}}},"buffer_pools":{"direct":{"count":111,"used_in_bytes":288779021,"total_capacity_in_bytes":288779020},"mapped":{"count":10210,"used_in_bytes":313880786534,"total_capacity_in_bytes":313880786534}},"classes":{"current_loaded_count":15315,"total_loaded_count":15459,"total_unloaded_count":144}},"thread_pool":{"bulk":{"threads":8,"queue":0,"active":0,"rejected":96063,"largest":8,"completed":6843393},"fetch_shard_started":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":2821},"fetch_shard_store":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":10417},"flush":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":235466},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":9,"queue":0,"active":0,"rejected":0,"largest":128,"completed":1203023},"get":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":30308},"index":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":1047},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":3,"rejected":0,"largest":5,"completed":1911510},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":122518082},"search":{"threads":13,"queue":0,"active":0,"rejected":0,"largest":13,"completed":647893},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":1799955},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":528310788096,"free_in_bytes":203463053312,"available_in_bytes":203446276096,"spins":"true"},"data":[{"path":"/mnt/data/nodes/0","mount":"/mnt/data (/dev/sdb1)","type":"ext4","total_in_bytes":528310788096,"free_in_bytes":203463053312,"available_in_bytes":203446276096,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sdb1","operations":35923102,"read_operations":5207659,"write_operations":30715443,"read_kilobytes":618782872,"write_kilobytes":2076833544}],"total":{"operations":35923102,"read_operations":5207659,"write_operations":30715443,"read_kilobytes":618782872,"write_kilobytes":2076833544}}},"transport":{"server_open":133,"rx_count":17156365,"rx_size_in_bytes":1032986412750,"tx_count":17159929,"tx_size_in_bytes":933277781744},"http":{"current_open":2,"total_opened":500007},"breakers":{"request":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":17110138880,"limit_size":"15.9gb","estimated_size_in_bytes":39799,"estimated_size":"38.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":11977097216,"limit_size":"11.1gb","estimated_size_in_bytes":39799,"estimated_size":"38.8kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"mr05ujLDTdq-vCnCNuCL8Q":{"timestamp":1511714498348,"name":"elasticcli-01","transport_address":"10.0.0.115:9300","host":"elasticcli-01","ip":"10.0.0.115:9300","roles":[],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714497945,"cpu":{"percent":8,"load_average":{"1m":0.3,"5m":0.24,"15m":0.27}},"mem":{"total_in_bytes":8375726080,"free_in_bytes":252256256,"used_in_bytes":8123469824,"free_percent":3,"used_percent":97},"swap":{"total_in_bytes":1073737728,"free_in_bytes":726077440,"used_in_bytes":347660288},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":30184458419459},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":518,"max_file_descriptors":1048576,"cpu":{"percent":2,"total_in_millis":30184300},"mem":{"total_virtual_in_bytes":6093246464}},"jvm":{"timestamp":1511714497945,"uptime_in_millis":445557994,"mem":{"heap_used_in_bytes":1203681360,"heap_used_percent":56,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":122900392,"non_heap_committed_in_bytes":130781184,"pools":{"young":{"used_in_bytes":192372408,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":34865152,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":976443800,"max_in_bytes":1798569984,"peak_used_in_bytes":1798569944,"peak_max_in_bytes":1798569984}}},"threads":{"count":63,"peak_count":64},"gc":{"collectors":{"young":{"collection_count":47488,"collection_time_in_millis":897897},"old":{"collection_count":544,"collection_time_in_millis":140989}}},"buffer_pools":{"direct":{"count":41,"used_in_bytes":152608776,"total_capacity_in_bytes":152608775},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":13664,"total_loaded_count":13736,"total_unloaded_count":72}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":578181},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":1268638},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":7,"queue":0,"active":0,"rejected":0,"largest":7,"completed":365840},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497945,"total":{"total_in_bytes":39836917760,"free_in_bytes":21528715264,"available_in_bytes":19481477120,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":21528715264,"available_in_bytes":19481477120,"spins":"true"}],"io_stats":{}},"transport":{"server_open":135,"rx_count":9063293,"rx_size_in_bytes":184591046718,"tx_count":9063303,"tx_size_in_bytes":82805398412},"http":{"current_open":14,"total_opened":410363},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":131520,"estimated_size":"128.4kb","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":131520,"estimated_size":"128.4kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"1rpMsJDnTFaSPdkBQM110g":{"timestamp":1511714498348,"name":"elasticdata-02","transport_address":"10.0.0.116:9300","host":"elasticdata-02","ip":"10.0.0.116:9300","roles":["data"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":408294223,"deleted":1074290},"store":{"size_in_bytes":257612662026,"throttle_time_in_millis":0},"indexing":{"index_total":568986485,"index_time_in_millis":135970122,"index_current":0,"index_failed":24361127,"delete_total":232,"delete_time_in_millis":29,"delete_current":0,"noop_update_total":113,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":15516,"time_in_millis":1369,"exists_total":15485,"exists_time_in_millis":1368,"missing_total":31,"missing_time_in_millis":1,"current":0},"search":{"open_contexts":0,"query_total":480400,"query_time_in_millis":4147683,"query_current":0,"fetch_total":25811,"fetch_time_in_millis":555392,"fetch_current":0,"scroll_total":19,"scroll_time_in_millis":1307018,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":199042,"total_time_in_millis":117076364,"total_docs":1647292265,"total_size_in_bytes":1026463665180,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":52910999,"total_auto_throttle_in_bytes":64807275536},"refresh":{"total":1782072,"total_time_in_millis":31882918,"listeners":0},"flush":{"total":2504,"total_time_in_millis":562585},"warmer":{"current":0,"total":1453580,"total_time_in_millis":429293},"query_cache":{"memory_size_in_bytes":167693059,"total_count":1256138,"hit_count":657110,"miss_count":599028,"cache_size":9740,"cache_count":11960,"evictions":2220},"fielddata":{"memory_size_in_bytes":46760,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4310,"memory_in_bytes":890541030,"terms_memory_in_bytes":798812291,"stored_fields_memory_in_bytes":63063768,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":8041856,"points_memory_in_bytes":4360995,"doc_values_memory_in_bytes":16262120,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":344,"max_unsafe_auto_id_timestamp":1511654404456,"file_sizes":{}},"translog":{"operations":2491457,"size_in_bytes":2453361251},"request_cache":{"memory_size_in_bytes":150055283,"evictions":16383,"hit_count":19785,"miss_count":38600},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":7657996}},"os":{"timestamp":1511714497945,"cpu":{"percent":9,"load_average":{"1m":0.88,"5m":0.92,"15m":0.87}},"mem":{"total_in_bytes":33742192640,"free_in_bytes":657498112,"used_in_bytes":33084694528,"free_percent":2,"used_percent":98},"swap":{"total_in_bytes":1073737728,"free_in_bytes":762204160,"used_in_bytes":311533568},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":473118153915722},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":1499,"max_file_descriptors":1048576,"cpu":{"percent":9,"total_in_millis":473115920},"mem":{"total_virtual_in_bytes":279946231808}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":584148847,"mem":{"heap_used_in_bytes":11282849256,"heap_used_percent":65,"heap_committed_in_bytes":17110138880,"heap_max_in_bytes":17110138880,"non_heap_used_in_bytes":182476568,"non_heap_committed_in_bytes":192057344,"pools":{"young":{"used_in_bytes":386452168,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":29119144,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":10867277944,"max_in_bytes":16481976320,"peak_used_in_bytes":16388546600,"peak_max_in_bytes":16481976320}}},"threads":{"count":135,"peak_count":195},"gc":{"collectors":{"young":{"collection_count":175896,"collection_time_in_millis":6408288},"old":{"collection_count":146,"collection_time_in_millis":58079}}},"buffer_pools":{"direct":{"count":136,"used_in_bytes":272251175,"total_capacity_in_bytes":272251174},"mapped":{"count":9822,"used_in_bytes":256092821621,"total_capacity_in_bytes":256092821621}},"classes":{"current_loaded_count":15342,"total_loaded_count":15513,"total_unloaded_count":171}},"thread_pool":{"bulk":{"threads":8,"queue":0,"active":0,"rejected":99626,"largest":8,"completed":5768105},"fetch_shard_started":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":3316},"fetch_shard_store":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":16,"completed":10424},"flush":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":232408},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":52,"completed":1355815},"get":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":15129},"index":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":477},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":3,"rejected":0,"largest":5,"completed":1911497},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":123430269},"search":{"threads":13,"queue":0,"active":0,"rejected":0,"largest":13,"completed":492351},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":1788033},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":528311836672,"free_in_bytes":259594153984,"available_in_bytes":232733884416,"spins":"true"},"data":[{"path":"/mnt/data/nodes/0","mount":"/mnt/data (/dev/sdb1)","type":"ext4","total_in_bytes":528311836672,"free_in_bytes":259594153984,"available_in_bytes":232733884416,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sdb1","operations":29499136,"read_operations":3305656,"write_operations":26193480,"read_kilobytes":412087356,"write_kilobytes":1819822768}],"total":{"operations":29499136,"read_operations":3305656,"write_operations":26193480,"read_kilobytes":412087356,"write_kilobytes":1819822768}}},"transport":{"server_open":133,"rx_count":15001696,"rx_size_in_bytes":1002526189002,"tx_count":15004766,"tx_size_in_bytes":1130024582664},"http":{"current_open":2,"total_opened":508442},"breakers":{"request":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":46760,"estimated_size":"45.6kb","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":17110138880,"limit_size":"15.9gb","estimated_size_in_bytes":78944,"estimated_size":"77kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":11977097216,"limit_size":"11.1gb","estimated_size_in_bytes":125704,"estimated_size":"122.7kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"mJtnri1LTJaymDIay6aJLQ":{"timestamp":1511714498349,"name":"elasticdata-06","transport_address":"10.0.0.97:9300","host":"elasticdata-06","ip":"10.0.0.97:9300","roles":["data"],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":458706036,"deleted":1011663},"store":{"size_in_bytes":305879051264,"throttle_time_in_millis":0},"indexing":{"index_total":352418731,"index_time_in_millis":96827787,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":166624,"time_in_millis":12473,"exists_total":166624,"exists_time_in_millis":12473,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":482343,"query_time_in_millis":2076387,"query_current":0,"fetch_total":169199,"fetch_time_in_millis":184224,"fetch_current":0,"scroll_total":2,"scroll_time_in_millis":240038,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":182114,"total_time_in_millis":127605140,"total_docs":1363877914,"total_size_in_bytes":960500345231,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":73083213,"total_auto_throttle_in_bytes":9823313645},"refresh":{"total":1567704,"total_time_in_millis":26218140,"listeners":0},"flush":{"total":1045,"total_time_in_millis":321661},"warmer":{"current":0,"total":1568611,"total_time_in_millis":696377},"query_cache":{"memory_size_in_bytes":80475461,"total_count":786337,"hit_count":444550,"miss_count":341787,"cache_size":8871,"cache_count":10021,"evictions":1150},"fielddata":{"memory_size_in_bytes":55288,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4465,"memory_in_bytes":1098390078,"terms_memory_in_bytes":996710745,"stored_fields_memory_in_bytes":72784232,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":8683072,"points_memory_in_bytes":4155561,"doc_values_memory_in_bytes":16056468,"index_writer_memory_in_bytes":911700,"version_map_memory_in_bytes":4650,"fixed_bit_set_memory_in_bytes":528,"max_unsafe_auto_id_timestamp":1511481600745,"file_sizes":{}},"translog":{"operations":3503036,"size_in_bytes":3357895432},"request_cache":{"memory_size_in_bytes":98505744,"evictions":0,"hit_count":26090,"miss_count":34723},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":1793510}},"os":{"timestamp":1511714497945,"cpu":{"percent":9,"load_average":{"1m":0.54,"5m":0.66,"15m":0.89}},"mem":{"total_in_bytes":33742192640,"free_in_bytes":329875456,"used_in_bytes":33412317184,"free_percent":1,"used_percent":99},"swap":{"total_in_bytes":1073737728,"free_in_bytes":755060736,"used_in_bytes":318676992},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":345265404353469},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":1347,"max_file_descriptors":1048576,"cpu":{"percent":8,"total_in_millis":345263060},"mem":{"total_virtual_in_bytes":326659825664}},"jvm":{"timestamp":1511714497946,"uptime_in_millis":431203379,"mem":{"heap_used_in_bytes":6856054400,"heap_used_percent":40,"heap_committed_in_bytes":17110138880,"heap_max_in_bytes":17110138880,"non_heap_used_in_bytes":168673048,"non_heap_committed_in_bytes":177786880,"pools":{"young":{"used_in_bytes":481488392,"max_in_bytes":558432256,"peak_used_in_bytes":558432256,"peak_max_in_bytes":558432256},"survivor":{"used_in_bytes":21141648,"max_in_bytes":69730304,"peak_used_in_bytes":69730304,"peak_max_in_bytes":69730304},"old":{"used_in_bytes":6353424360,"max_in_bytes":16481976320,"peak_used_in_bytes":12677713896,"peak_max_in_bytes":16481976320}}},"threads":{"count":128,"peak_count":135},"gc":{"collectors":{"young":{"collection_count":127223,"collection_time_in_millis":4468108},"old":{"collection_count":35,"collection_time_in_millis":13393}}},"buffer_pools":{"direct":{"count":125,"used_in_bytes":272179208,"total_capacity_in_bytes":272179207},"mapped":{"count":10044,"used_in_bytes":302869020633,"total_capacity_in_bytes":302869020633}},"classes":{"current_loaded_count":15194,"total_loaded_count":15397,"total_unloaded_count":203}},"thread_pool":{"bulk":{"threads":8,"queue":0,"active":0,"rejected":656,"largest":8,"completed":5424968},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":172578},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":7,"queue":0,"active":0,"rejected":0,"largest":7,"completed":981787},"get":{"threads":8,"queue":0,"active":0,"rejected":0,"largest":8,"completed":166535},"index":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":4},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":2,"rejected":0,"largest":5,"completed":1391304},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":95745951},"search":{"threads":13,"queue":0,"active":0,"rejected":0,"largest":13,"completed":485078},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":1569132},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":528310788096,"free_in_bytes":218908598272,"available_in_bytes":218891821056,"spins":"true"},"data":[{"path":"/mnt/data/nodes/0","mount":"/mnt/data (/dev/sdb1)","type":"ext4","total_in_bytes":528310788096,"free_in_bytes":218908598272,"available_in_bytes":218891821056,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sdb1","operations":25942355,"read_operations":1681165,"write_operations":24261190,"read_kilobytes":193190936,"write_kilobytes":1621218228}],"total":{"operations":25942355,"read_operations":1681165,"write_operations":24261190,"read_kilobytes":193190936,"write_kilobytes":1621218228}}},"transport":{"server_open":133,"rx_count":12260547,"rx_size_in_bytes":541703802431,"tx_count":12260545,"tx_size_in_bytes":346776082145},"http":{"current_open":2,"total_opened":381606},"breakers":{"request":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":10266083328,"limit_size":"9.5gb","estimated_size_in_bytes":55288,"estimated_size":"53.9kb","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":17110138880,"limit_size":"15.9gb","estimated_size_in_bytes":37927,"estimated_size":"37kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":11977097216,"limit_size":"11.1gb","estimated_size_in_bytes":93215,"estimated_size":"91kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}},"UM8bpuowTXq2xX5B_w96jg":{"timestamp":1511714498349,"name":"elasticcli-02","transport_address":"10.0.0.114:9300","host":"elasticcli-02","ip":"10.0.0.114:9300","roles":[],"attributes":{"ml.enabled":"true"},"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":0,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":0,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-9223372036854775808,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1511714497945,"cpu":{"percent":8,"load_average":{"1m":0.11,"5m":0.29,"15m":0.38}},"mem":{"total_in_bytes":8375726080,"free_in_bytes":242339840,"used_in_bytes":8133386240,"free_percent":3,"used_percent":97},"swap":{"total_in_bytes":1073737728,"free_in_bytes":666390528,"used_in_bytes":407347200},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":24933856576484},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1511714497945,"open_file_descriptors":516,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":24933720},"mem":{"total_virtual_in_bytes":6057439232}},"jvm":{"timestamp":1511714497945,"uptime_in_millis":439494501,"mem":{"heap_used_in_bytes":630212256,"heap_used_percent":29,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":123008888,"non_heap_committed_in_bytes":130932736,"pools":{"young":{"used_in_bytes":43037928,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":28583072,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":558591256,"max_in_bytes":1798569984,"peak_used_in_bytes":1783141720,"peak_max_in_bytes":1798569984}}},"threads":{"count":59,"peak_count":62},"gc":{"collectors":{"young":{"collection_count":39584,"collection_time_in_millis":605986},"old":{"collection_count":200,"collection_time_in_millis":25256}}},"buffer_pools":{"direct":{"count":40,"used_in_bytes":135831560,"total_capacity_in_bytes":135831559},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":13655,"total_loaded_count":13724,"total_unloaded_count":69}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":570734},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":5,"queue":0,"active":1,"rejected":0,"largest":5,"completed":1243282},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":7,"queue":0,"active":0,"rejected":0,"largest":7,"completed":339462},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1511714497946,"total":{"total_in_bytes":39836917760,"free_in_bytes":21595881472,"available_in_bytes":19548643328,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":39836917760,"free_in_bytes":21595881472,"available_in_bytes":19548643328,"spins":"true"}],"io_stats":{}},"transport":{"server_open":135,"rx_count":8721285,"rx_size_in_bytes":169344861699,"tx_count":8721284,"tx_size_in_bytes":73698352750},"http":{"current_open":11,"total_opened":403280},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":1916,"estimated_size":"1.8kb","overhead":1.0,"tripped":0}},"script":{"compilations":1,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}}}}`, } for ver, out := range tcs { for hn, handler := range map[string]http.Handler{ "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, out) }), "basicauth": &basicAuth{ User: "elastic", Pass: "changeme", Next: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, out) }), }, } { ts := httptest.NewServer(handler) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("Failed to parse URL: %s", err) } u.User = url.UserPassword("elastic", "changeme") c := NewNodes(log.NewNopLogger(), http.DefaultClient, u, true, "_local") nsr, err := c.fetchAndDecodeNodeStats() if err != nil { t.Fatalf("Failed to fetch or decode node stats: %s", err) } t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) if nsr.ClusterName == "elasticsearch" { for _, nsnr := range nsr.Nodes { if nsnr.Indices.Docs.Count > 0 { t.Errorf("Wrong doc count") } } } if nsr.ClusterName == "multinode" { for _, node := range nsr.Nodes { labels := defaultNodeLabelValues(nsr.ClusterName, node) esMasterNode := labels[3] esDataNode := labels[4] esIngestNode := labels[5] esClientNode := labels[6] t.Logf( "Node: %s - Master: %s - Data: %s - Ingest: %s - Client: %s", node.Name, esMasterNode, esDataNode, esIngestNode, esClientNode, ) if strings.HasPrefix(node.Name, "elasticmaster") { if esMasterNode != "true" { t.Errorf("Master should be master") } if esDataNode == "true" { t.Errorf("Master should be not data") } if esIngestNode == "true" { t.Errorf("Master should be not ingest") } } if strings.HasPrefix(node.Name, "elasticdata") { if esMasterNode == "true" { t.Errorf("Data should not be master") } if esDataNode != "true" { t.Errorf("Data should be data") } if esIngestNode == "true" { t.Errorf("Data should be not ingest") } } if strings.HasPrefix(node.Name, "elasticin") { if esMasterNode == "true" { t.Errorf("Ingest should not be master") } if esDataNode == "true" { t.Errorf("Ingest should be data") } if esIngestNode != "true" { t.Errorf("Ingest should be not ingest") } } if strings.HasPrefix(node.Name, "elasticcli") { if esMasterNode == "true" { t.Errorf("CLI should not be master") } if esDataNode == "true" { t.Errorf("CLI should be data") } if esIngestNode == "true" { t.Errorf("CLI should be not ingest") } } } } } } } type basicAuth struct { User string Pass string Next http.Handler } func (h *basicAuth) checkAuth(w http.ResponseWriter, r *http.Request) bool { s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) if len(s) != 2 { return false } b, err := base64.StdEncoding.DecodeString(s[1]) if err != nil { return false } pair := strings.SplitN(string(b), ":", 2) if len(pair) != 2 { return false } if h.User == pair[0] && h.Pass == pair[1] { return true } return false } func (h *basicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !h.checkAuth(w, r) { w.Header().Set("WWW-Authenticate", "Basic realm=\"ES\"") w.WriteHeader(401) w.Write([]byte("401 Unauthorized\n")) return } h.Next.ServeHTTP(w, r) } prometheus-elasticsearch-exporter-1.1.0+ds/collector/snapshots.go000066400000000000000000000206551362377146000253440ustar00rootroot00000000000000package collector import ( "encoding/json" "fmt" "net/http" "net/url" "path" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) type snapshotMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(snapshotStats SnapshotStatDataResponse) float64 Labels func(repositoryName string, snapshotStats SnapshotStatDataResponse) []string } type repositoryMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(snapshotsStats SnapshotStatsResponse) float64 Labels func(repositoryName string) []string } var ( defaultSnapshotLabels = []string{"repository", "state", "version"} defaultSnapshotLabelValues = func(repositoryName string, snapshotStats SnapshotStatDataResponse) []string { return []string{repositoryName, snapshotStats.State, snapshotStats.Version} } defaultSnapshotRepositoryLabels = []string{"repository"} defaultSnapshotRepositoryLabelValues = func(repositoryName string) []string { return []string{repositoryName} } ) // Snapshots information struct type Snapshots struct { logger log.Logger client *http.Client url *url.URL up prometheus.Gauge totalScrapes, jsonParseFailures prometheus.Counter snapshotMetrics []*snapshotMetric repositoryMetrics []*repositoryMetric } // NewSnapshots defines Snapshots Prometheus metrics func NewSnapshots(logger log.Logger, client *http.Client, url *url.URL) *Snapshots { return &Snapshots{ logger: logger, client: client, url: url, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "snapshot_stats", "up"), Help: "Was the last scrape of the ElasticSearch snapshots endpoint successful.", }), totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "snapshot_stats", "total_scrapes"), Help: "Current total ElasticSearch snapshots scrapes.", }), jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ Name: prometheus.BuildFQName(namespace, "snapshot_stats", "json_parse_failures"), Help: "Number of errors while parsing JSON.", }), snapshotMetrics: []*snapshotMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_number_of_indices"), "Number of indices in the last snapshot", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(len(snapshotStats.Indices)) }, Labels: defaultSnapshotLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_start_time_timestamp"), "Last snapshot start timestamp", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(snapshotStats.StartTimeInMillis / 1000) }, Labels: defaultSnapshotLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_end_time_timestamp"), "Last snapshot end timestamp", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(snapshotStats.EndTimeInMillis / 1000) }, Labels: defaultSnapshotLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_number_of_failures"), "Last snapshot number of failures", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(len(snapshotStats.Failures)) }, Labels: defaultSnapshotLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_total_shards"), "Last snapshot total shards", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(snapshotStats.Shards.Total) }, Labels: defaultSnapshotLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_failed_shards"), "Last snapshot failed shards", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(snapshotStats.Shards.Failed) }, Labels: defaultSnapshotLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "snapshot_successful_shards"), "Last snapshot successful shards", defaultSnapshotLabels, nil, ), Value: func(snapshotStats SnapshotStatDataResponse) float64 { return float64(snapshotStats.Shards.Successful) }, Labels: defaultSnapshotLabelValues, }, }, repositoryMetrics: []*repositoryMetric{ { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "number_of_snapshots"), "Number of snapshots in a repository", defaultSnapshotRepositoryLabels, nil, ), Value: func(snapshotsStats SnapshotStatsResponse) float64 { return float64(len(snapshotsStats.Snapshots)) }, Labels: defaultSnapshotRepositoryLabelValues, }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "snapshot_stats", "oldest_snapshot_timestamp"), "Timestamp of the oldest snapshot", defaultSnapshotRepositoryLabels, nil, ), Value: func(snapshotsStats SnapshotStatsResponse) float64 { if len(snapshotsStats.Snapshots) == 0 { return 0 } return float64(snapshotsStats.Snapshots[0].StartTimeInMillis / 1000) }, Labels: defaultSnapshotRepositoryLabelValues, }, }, } } // Describe add Snapshots metrics descriptions func (s *Snapshots) Describe(ch chan<- *prometheus.Desc) { for _, metric := range s.snapshotMetrics { ch <- metric.Desc } ch <- s.up.Desc() ch <- s.totalScrapes.Desc() ch <- s.jsonParseFailures.Desc() } func (s *Snapshots) getAndParseURL(u *url.URL, data interface{}) error { res, err := s.client.Get(u.String()) if err != nil { return fmt.Errorf("failed to get from %s://%s:%s%s: %s", u.Scheme, u.Hostname(), u.Port(), u.Path, err) } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(s.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(data); err != nil { s.jsonParseFailures.Inc() return err } return nil } func (s *Snapshots) fetchAndDecodeSnapshotsStats() (map[string]SnapshotStatsResponse, error) { mssr := make(map[string]SnapshotStatsResponse) u := *s.url u.Path = path.Join(u.Path, "/_snapshot") var srr SnapshotRepositoriesResponse err := s.getAndParseURL(&u, &srr) if err != nil { return nil, err } for repository := range srr { u := *s.url u.Path = path.Join(u.Path, "/_snapshot", repository, "/_all") var ssr SnapshotStatsResponse err := s.getAndParseURL(&u, &ssr) if err != nil { continue } mssr[repository] = ssr } return mssr, nil } // Collect gets Snapshots metric values func (s *Snapshots) Collect(ch chan<- prometheus.Metric) { s.totalScrapes.Inc() defer func() { ch <- s.up ch <- s.totalScrapes ch <- s.jsonParseFailures }() // indices snapshotsStatsResp, err := s.fetchAndDecodeSnapshotsStats() if err != nil { s.up.Set(0) _ = level.Warn(s.logger).Log( "msg", "failed to fetch and decode snapshot stats", "err", err, ) return } s.up.Set(1) // Snapshots stats for repositoryName, snapshotStats := range snapshotsStatsResp { for _, metric := range s.repositoryMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(snapshotStats), metric.Labels(repositoryName)..., ) } if len(snapshotStats.Snapshots) == 0 { continue } lastSnapshot := snapshotStats.Snapshots[len(snapshotStats.Snapshots)-1] for _, metric := range s.snapshotMetrics { ch <- prometheus.MustNewConstMetric( metric.Desc, metric.Type, metric.Value(lastSnapshot), metric.Labels(repositoryName, lastSnapshot)..., ) } } } prometheus-elasticsearch-exporter-1.1.0+ds/collector/snapshots_reponse.go000066400000000000000000000024611362377146000270720ustar00rootroot00000000000000package collector import "time" // SnapshotStatsResponse is a representation of the snapshots stats type SnapshotStatsResponse struct { Snapshots []SnapshotStatDataResponse `json:"snapshots"` } // SnapshotStatDataResponse is a representation of the single snapshot stat type SnapshotStatDataResponse struct { Snapshot string `json:"snapshot"` UUID string `json:"uuid"` VersionID int64 `json:"version_id"` Version string `json:"version"` Indices []string `json:"indices"` State string `json:"state"` StartTime time.Time `json:"start_time"` StartTimeInMillis int64 `json:"start_time_in_millis"` EndTime time.Time `json:"end_time"` EndTimeInMillis int64 `json:"end_time_in_millis"` DurationInMillis int64 `json:"duration_in_millis"` Failures []interface{} `json:"failures"` Shards struct { Total int64 `json:"total"` Failed int64 `json:"failed"` Successful int64 `json:"successful"` } `json:"shards"` } // SnapshotRepositoriesResponse is a representation snapshots repositories type SnapshotRepositoriesResponse map[string]struct { Type string `json:"type"` Settings map[string]string `json:"settings"` } prometheus-elasticsearch-exporter-1.1.0+ds/collector/snapshots_test.go000066400000000000000000000107311362377146000263750ustar00rootroot00000000000000package collector import ( "fmt" "net/http" "net/http/httptest" "net/url" "testing" "github.com/go-kit/kit/log" ) func TestSnapshots(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine -Des.path.repo="/tmp" (1.7.6, 2.4.5) // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine -E path.repo="/tmp" (5.4.2) // curl -XPUT http://localhost:9200/foo_1/type1/1 -d '{"title":"abc","content":"hello"}' // curl -XPUT http://localhost:9200/foo_1/type1/2 -d '{"title":"def","content":"world"}' // curl -XPUT http://localhost:9200/foo_2/type1/1 -d '{"title":"abc001","content":"hello001"}' // curl -XPUT http://localhost:9200/foo_2/type1/2 -d '{"title":"def002","content":"world002"}' // curl -XPUT http://localhost:9200/foo_2/type1/3 -d '{"title":"def003","content":"world003"}' // curl -XPUT http://localhost:9200/_snapshot/test1 -d '{"type": "fs","settings":{"location": "/tmp/test1"}}' // curl -XPUT "http://localhost:9200/_snapshot/test1/snapshot_1?wait_for_completion=true" // curl http://localhost:9200/_snapshot/ // curl http://localhost:9200/_snapshot/test1/_all tcs := map[string][]string{ "1.7.6": {`{"test1":{"type":"fs","settings":{"location":"/tmp/test1"}}}`, `{"snapshots":[{"snapshot":"snapshot_1","version_id":1070699,"version":"1.7.6","indices":["foo_1","foo_2"],"state":"SUCCESS","start_time":"2018-09-04T09:09:02.427Z","start_time_in_millis":1536052142427,"end_time":"2018-09-04T09:09:02.755Z","end_time_in_millis":1536052142755,"duration_in_millis":328,"failures":[],"shards":{"total":10,"failed":0,"successful":10}}]}`}, "2.4.5": {`{"test1":{"type":"fs","settings":{"location":"/tmp/test1"}}}`, `{"snapshots":[{"snapshot":"snapshot_1","version_id":2040599,"version":"2.4.5","indices":["foo_2","foo_1"],"state":"SUCCESS","start_time":"2018-09-04T09:25:25.818Z","start_time_in_millis":1536053125818,"end_time":"2018-09-04T09:25:26.326Z","end_time_in_millis":1536053126326,"duration_in_millis":508,"failures":[],"shards":{"total":10,"failed":0,"successful":10}}]}`}, "5.4.2": {`{"test1":{"type":"fs","settings":{"location":"/tmp/test1"}}}`, `{"snapshots":[{"snapshot":"snapshot_1","uuid":"VZ_c_kKISAW8rpcqiwSg0w","version_id":5040299,"version":"5.4.2","indices":["foo_2","foo_1"],"state":"SUCCESS","start_time":"2018-09-04T09:29:13.971Z","start_time_in_millis":1536053353971,"end_time":"2018-09-04T09:29:14.477Z","end_time_in_millis":1536053354477,"duration_in_millis":506,"failures":[],"shards":{"total":10,"failed":0,"successful":10}}]}`}, "5.4.2-failed": {`{"test1":{"type":"fs","settings":{"location":"/tmp/test1"}}}`, `{"snapshots":[{"snapshot":"snapshot_1","uuid":"VZ_c_kKISAW8rpcqiwSg0w","version_id":5040299,"version":"5.4.2","indices":["foo_2","foo_1"],"state":"SUCCESS","start_time":"2018-09-04T09:29:13.971Z","start_time_in_millis":1536053353971,"end_time":"2018-09-04T09:29:14.477Z","end_time_in_millis":1536053354477,"duration_in_millis":506,"failures":[{"index" : "index_name","index_uuid" : "index_name","shard_id" : 52,"reason" : "IndexShardSnapshotFailedException[error deleting index file [pending-index-5] during cleanup]; nested: NoSuchFileException[Blob [pending-index-5] does not exist]; ","node_id" : "pPm9jafyTjyMk0T5A101xA","status" : "INTERNAL_SERVER_ERROR"}],"shards":{"total":10,"failed":1,"successful":10}}]}`}, } for ver, out := range tcs { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.RequestURI == "/_snapshot" { fmt.Fprint(w, out[0]) return } fmt.Fprint(w, out[1]) })) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("Failed to parse URL: %s", err) } s := NewSnapshots(log.NewNopLogger(), http.DefaultClient, u) stats, err := s.fetchAndDecodeSnapshotsStats() if err != nil { t.Fatalf("Failed to fetch or decode snapshots stats: %s", err) } t.Logf("[%s] Snapshots Response: %+v", ver, stats) repositoryStats := stats["test1"] snapshotStats := repositoryStats.Snapshots[0] if len(snapshotStats.Indices) != 2 { t.Errorf("Bad number of snapshot indices") } if len(snapshotStats.Failures) != int(snapshotStats.Shards.Failed) { t.Errorf("Bad number of snapshot failures") } if snapshotStats.Shards.Total != 10 { t.Errorf("Bad number of snapshot shards total") } if snapshotStats.Shards.Successful != 10 { t.Errorf("Bad number of snapshot shards successful") } if len(repositoryStats.Snapshots) != 1 { t.Errorf("Bad number of repository snapshots") } } } prometheus-elasticsearch-exporter-1.1.0+ds/examples/000077500000000000000000000000001362377146000226135ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/examples/grafana/000077500000000000000000000000001362377146000242125ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/examples/grafana/dashboard.json000066400000000000000000001407241362377146000270440ustar00rootroot00000000000000{ "__inputs": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "5.4.0" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "5.0.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "5.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", "version": "5.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "id": null, "iteration": 1549021227642, "links": [], "panels": [ { "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 90, "title": "Cluster", "type": "row" }, { "cacheTimeout": null, "colorBackground": true, "colorPostfix": false, "colorPrefix": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 12, "x": 0, "y": 1 }, "id": 92, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "tableColumn": "Value", "targets": [ { "expr": "scalar(elasticsearch_cluster_health_status{color=\"green\",cluster=~\"$cluster\"}) + scalar(elasticsearch_cluster_health_status{color=\"yellow\",cluster=~\"$cluster\"}) * 2 + scalar(elasticsearch_cluster_health_status{color=\"red\",cluster=~\"$cluster\"}) * 3", "format": "time_series", "instant": false, "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "thresholds": "2,3", "title": "Cluster Status", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" }, { "op": "=", "text": "green", "value": "1" }, { "op": "=", "text": "yellow", "value": "2" }, { "op": "=", "text": "red", "value": "3" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$server", "editable": true, "error": false, "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 }, "id": 8, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})/count(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "metric": "elasticsearch_cluster_health_number_of_nodes", "refId": "A", "step": 1800 } ], "thresholds": "", "timeFrom": null, "timeShift": null, "title": "Running Nodes", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 }, "id": 94, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Active Data Nodes", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, "id": 96, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "0.5,1", "title": "Pending Tasks", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 76, "panels": [], "title": "Shards", "type": "row" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 5 }, "id": 78, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Active Shards", "transparent": false, "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 5 }, "id": 80, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "", "title": "Active Primary Shards", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 5 }, "id": 82, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "0.5,1", "title": "Initializing Shards", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 12, "y": 5 }, "id": 84, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_relocating_shards{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "0.5,1", "title": "Relocating Shards", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 16, "y": 5 }, "id": 86, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "0.5,1", "title": "Unassigned Shards", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": true, "colors": [ "#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a" ], "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 5 }, "id": 88, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}", "format": "time_series", "intervalFactor": 1, "refId": "A" } ], "thresholds": "0.5,1", "title": "Delayed Unassigned Shards", "type": "singlestat", "valueFontSize": "120%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 70, "panels": [], "title": "Documents", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 9 }, "id": 3, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(elasticsearch_indices_docs{cluster=~\"$cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Documents", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Documents indexed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 9 }, "id": 4, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Index Size", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 18 }, "id": 72, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[1h])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Documents Indexed Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Documents/s", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 18 }, "id": 74, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(elasticsearch_indices_search_fetch_total{cluster=~\"$cluster\"}[1h])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Queris/s", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 27 }, "height": "", "id": 64, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", "interval": "", "intervalFactor": 2, "legendFormat": "Type: {{type}}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Queue Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 68, "panels": [], "title": "System", "type": "row" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$server", "editable": true, "error": false, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 8, "x": 0, "y": 36 }, "height": "", "id": 12, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(node_memory_MemTotal{cluster=~\"$cluster\"})", "intervalFactor": 2, "legendFormat": "", "metric": "", "refId": "A", "step": 1800 } ], "thresholds": "", "title": "Total Memory", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$server", "editable": true, "error": false, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 8, "x": 8, "y": 36 }, "height": "", "id": 13, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(node_memory_MemFree{cluster=~\"$cluster\"})", "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 1800 } ], "thresholds": "", "title": "Total Memory Free", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "$server", "editable": true, "error": false, "format": "bytes", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 3, "w": 8, "x": 16, "y": 36 }, "height": "", "id": 14, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "expr": "sum(node_memory_MemAvailable{cluster=~\"$cluster\"})", "intervalFactor": 2, "legendFormat": "", "refId": "A", "step": 1800 } ], "thresholds": "", "title": "Total Memory Available", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, "id": 1, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(elasticsearch_thread_pool_active_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Type: {{ type }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread Pools", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 39 }, "id": 66, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\", type!=\"management\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{ name }} {{ type }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool rejections", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "fill": 1, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 47 }, "id": 57, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "dsType": "elasticsearch", "expr": "avg(irate(node_cpu{cluster=~\"$cluster\"}[10s])) by(mode) *100", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ mode }}", "metric": "elasticsearch_breakers_tripped", "metrics": [ { "id": "1", "type": "count" } ], "refId": "A", "step": 240, "timeField": "failure_tstamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg. CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 47 }, "id": 28, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg_over_time(elasticsearch_jvm_memory_used_bytes{area=\"heap\",cluster=~\"$cluster\"}[15m]) / elasticsearch_jvm_memory_max_bytes{area=\"heap\",cluster=~\"$cluster\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ name }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg Heap in 15min", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": "", "logBase": 1, "max": 1, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, "id": 5, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(elasticsearch_transport_rx_packets_total{cluster=~\"$cluster\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "RX", "refId": "A", "step": 240 }, { "expr": "sum(rate(elasticsearch_transport_tx_packets_total{cluster=~\"$cluster\"}[5m])) * -1", "format": "time_series", "intervalFactor": 2, "legendFormat": "TX", "refId": "B", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RX/TX Rate 5m", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$server", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, "id": 65, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "irate(elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[1m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{ name }} {{ gc }}", "refId": "A", "step": 240 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "GC seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "1m", "schemaVersion": 16, "style": "dark", "tags": [ "infra", "b2c", "elastic" ], "templating": { "list": [ { "current": { "text": "Prometheus", "value": "Prometheus" }, "hide": 0, "label": "Server", "name": "server", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": {}, "datasource": "$server", "definition": "", "hide": 0, "includeAll": true, "label": null, "multi": true, "name": "cluster", "options": [], "query": "label_values(elasticsearch_cluster_health_status,cluster)", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "utc", "title": "ElasticSearch Example", "uid": "n_nxrE_mk", "version": 1 } prometheus-elasticsearch-exporter-1.1.0+ds/examples/kubernetes/000077500000000000000000000000001362377146000247625ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/examples/kubernetes/deployment.yml000066400000000000000000000031441362377146000276670ustar00rootroot00000000000000apiVersion: apps/v1 kind: Deployment metadata: name: p8s-elastic-exporter spec: replicas: 1 strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 0 type: RollingUpdate selector: matchLabels: app: p8s-elastic-exporter template: metadata: labels: app: p8s-elastic-exporter spec: securityContext: runAsNonRoot: true runAsUser: 1000 containers: - command: - /bin/elasticsearch_exporter - -es.uri=http://elasticsearch:9200 - -es.all=true - -web.listen-address=:9114 image: justwatch/elasticsearch_exporter:1.0.2 securityContext: capabilities: drop: - SETPCAP - MKNOD - AUDIT_WRITE - CHOWN - NET_RAW - DAC_OVERRIDE - FOWNER - FSETID - KILL - SETGID - SETUID - NET_BIND_SERVICE - SYS_CHROOT - SETFCAP readOnlyRootFilesystem: true livenessProbe: httpGet: path: /health port: 9114 initialDelaySeconds: 30 timeoutSeconds: 10 name: p8s-elastic-exporter ports: - containerPort: 9114 readinessProbe: httpGet: path: /health port: 9114 initialDelaySeconds: 10 timeoutSeconds: 10 resources: limits: cpu: 100m memory: 128Mi requests: cpu: 25m memory: 64Mi restartPolicy: Always prometheus-elasticsearch-exporter-1.1.0+ds/examples/prometheus/000077500000000000000000000000001362377146000250065ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/examples/prometheus/elasticsearch.rules000066400000000000000000000017231362377146000306770ustar00rootroot00000000000000# calculate filesystem used and free percent elasticsearch_filesystem_data_used_percent = 100 * (elasticsearch_filesystem_data_size_bytes - elasticsearch_filesystem_data_free_bytes) / elasticsearch_filesystem_data_size_bytes elasticsearch_filesystem_data_free_percent = 100 - elasticsearch_filesystem_data_used_percent # alert if too few nodes are running ALERT ElasticsearchTooFewNodesRunning IF elasticsearch_cluster_health_number_of_nodes < 3 FOR 5m LABELS {severity="critical"} ANNOTATIONS {description="There are only {{$value}} < 3 ElasticSearch nodes running", summary="ElasticSearch running on less than 3 nodes"} # alert if heap usage is over 90% ALERT ElasticsearchHeapTooHigh IF elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} > 0.9 FOR 15m LABELS {severity="critical"} ANNOTATIONS {description="The heap usage is over 90% for 15m", summary="ElasticSearch node {{$labels.node}} heap usage is high"} prometheus-elasticsearch-exporter-1.1.0+ds/examples/prometheus/elasticsearch.rules.yml000066400000000000000000000017621362377146000315020ustar00rootroot00000000000000groups: - name: elasticsearch rules: - record: elasticsearch_filesystem_data_used_percent expr: 100 * (elasticsearch_filesystem_data_size_bytes - elasticsearch_filesystem_data_free_bytes) / elasticsearch_filesystem_data_size_bytes - record: elasticsearch_filesystem_data_free_percent expr: 100 - elasticsearch_filesystem_data_used_percent - alert: ElasticsearchTooFewNodesRunning expr: elasticsearch_cluster_health_number_of_nodes < 3 for: 5m labels: severity: critical annotations: description: There are only {{$value}} < 3 ElasticSearch nodes running summary: ElasticSearch running on less than 3 nodes - alert: ElasticsearchHeapTooHigh expr: elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} > 0.9 for: 15m labels: severity: critical annotations: description: The heap usage is over 90% for 15m summary: ElasticSearch node {{$labels.node}} heap usage is high prometheus-elasticsearch-exporter-1.1.0+ds/logger.go000066400000000000000000000021751362377146000226100ustar00rootroot00000000000000package main import ( "os" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "io" "strings" ) func getLogger(loglevel, logoutput, logfmt string) log.Logger { var out *os.File switch strings.ToLower(logoutput) { case "stderr": out = os.Stderr case "stdout": out = os.Stdout default: out = os.Stdout } var logCreator func(io.Writer) log.Logger switch strings.ToLower(logfmt) { case "json": logCreator = log.NewJSONLogger case "logfmt": logCreator = log.NewLogfmtLogger default: logCreator = log.NewLogfmtLogger } // create a logger logger := logCreator(log.NewSyncWriter(out)) // set loglevel var loglevelFilterOpt level.Option switch strings.ToLower(loglevel) { case "debug": loglevelFilterOpt = level.AllowDebug() case "info": loglevelFilterOpt = level.AllowInfo() case "warn": loglevelFilterOpt = level.AllowWarn() case "error": loglevelFilterOpt = level.AllowError() default: loglevelFilterOpt = level.AllowInfo() } logger = level.NewFilter(logger, loglevelFilterOpt) logger = log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller, ) return logger } prometheus-elasticsearch-exporter-1.1.0+ds/main.go000066400000000000000000000162411362377146000222540ustar00rootroot00000000000000package main import ( "net/http" "net/url" "os" "os/signal" "time" "context" "github.com/go-kit/kit/log/level" "github.com/justwatchcom/elasticsearch_exporter/collector" "github.com/justwatchcom/elasticsearch_exporter/pkg/clusterinfo" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" "gopkg.in/alecthomas/kingpin.v2" ) func main() { var ( Name = "elasticsearch_exporter" listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry."). Default(":9114").Envar("WEB_LISTEN_ADDRESS").String() metricsPath = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics."). Default("/metrics").Envar("WEB_TELEMETRY_PATH").String() esURI = kingpin.Flag("es.uri", "HTTP API address of an Elasticsearch node."). Default("http://localhost:9200").Envar("ES_URI").String() esTimeout = kingpin.Flag("es.timeout", "Timeout for trying to get stats from Elasticsearch."). Default("5s").Envar("ES_TIMEOUT").Duration() esAllNodes = kingpin.Flag("es.all", "Export stats for all nodes in the cluster. If used, this flag will override the flag es.node."). Default("false").Envar("ES_ALL").Bool() esNode = kingpin.Flag("es.node", "Node's name of which metrics should be exposed."). Default("_local").Envar("ES_NODE").String() esExportIndices = kingpin.Flag("es.indices", "Export stats for indices in the cluster."). Default("false").Envar("ES_INDICES").Bool() esExportIndicesSettings = kingpin.Flag("es.indices_settings", "Export stats for settings of all indices of the cluster."). Default("false").Envar("ES_INDICES_SETTINGS").Bool() esExportClusterSettings = kingpin.Flag("es.cluster_settings", "Export stats for cluster settings."). Default("false").Envar("ES_CLUSTER_SETTINGS").Bool() esExportShards = kingpin.Flag("es.shards", "Export stats for shards in the cluster (implies --es.indices)."). Default("false").Envar("ES_SHARDS").Bool() esExportSnapshots = kingpin.Flag("es.snapshots", "Export stats for the cluster snapshots."). Default("false").Envar("ES_SNAPSHOTS").Bool() esClusterInfoInterval = kingpin.Flag("es.clusterinfo.interval", "Cluster info update interval for the cluster label"). Default("5m").Envar("ES_CLUSTERINFO_INTERVAL").Duration() esCA = kingpin.Flag("es.ca", "Path to PEM file that contains trusted Certificate Authorities for the Elasticsearch connection."). Default("").Envar("ES_CA").String() esClientPrivateKey = kingpin.Flag("es.client-private-key", "Path to PEM file that contains the private key for client auth when connecting to Elasticsearch."). Default("").Envar("ES_CLIENT_PRIVATE_KEY").String() esClientCert = kingpin.Flag("es.client-cert", "Path to PEM file that contains the corresponding cert for the private key to connect to Elasticsearch."). Default("").Envar("ES_CLIENT_CERT").String() esInsecureSkipVerify = kingpin.Flag("es.ssl-skip-verify", "Skip SSL verification when connecting to Elasticsearch."). Default("false").Envar("ES_SSL_SKIP_VERIFY").Bool() logLevel = kingpin.Flag("log.level", "Sets the loglevel. Valid levels are debug, info, warn, error"). Default("info").Envar("LOG_LEVEL").String() logFormat = kingpin.Flag("log.format", "Sets the log format. Valid formats are json and logfmt"). Default("logfmt").Envar("LOG_FMT").String() logOutput = kingpin.Flag("log.output", "Sets the log output. Valid outputs are stdout and stderr"). Default("stdout").Envar("LOG_OUTPUT").String() ) kingpin.Version(version.Print(Name)) kingpin.CommandLine.HelpFlag.Short('h') kingpin.Parse() logger := getLogger(*logLevel, *logOutput, *logFormat) esURL, err := url.Parse(*esURI) if err != nil { _ = level.Error(logger).Log( "msg", "failed to parse es.uri", "err", err, ) os.Exit(1) } // returns nil if not provided and falls back to simple TCP. tlsConfig := createTLSConfig(*esCA, *esClientCert, *esClientPrivateKey, *esInsecureSkipVerify) httpClient := &http.Client{ Timeout: *esTimeout, Transport: &http.Transport{ TLSClientConfig: tlsConfig, Proxy: http.ProxyFromEnvironment, }, } // version metric versionMetric := version.NewCollector(Name) prometheus.MustRegister(versionMetric) // cluster info retriever clusterInfoRetriever := clusterinfo.New(logger, httpClient, esURL, *esClusterInfoInterval) prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes, *esNode)) if *esExportIndices || *esExportShards { iC := collector.NewIndices(logger, httpClient, esURL, *esExportShards) prometheus.MustRegister(iC) if registerErr := clusterInfoRetriever.RegisterConsumer(iC); registerErr != nil { _ = level.Error(logger).Log("msg", "failed to register indices collector in cluster info") os.Exit(1) } } if *esExportSnapshots { prometheus.MustRegister(collector.NewSnapshots(logger, httpClient, esURL)) } if *esExportClusterSettings { prometheus.MustRegister(collector.NewClusterSettings(logger, httpClient, esURL)) } if *esExportIndicesSettings { prometheus.MustRegister(collector.NewIndicesSettings(logger, httpClient, esURL)) } // create a http server server := &http.Server{} // create a context that is cancelled on SIGKILL ctx, cancel := context.WithCancel(context.Background()) // start the cluster info retriever switch runErr := clusterInfoRetriever.Run(ctx); runErr { case nil: _ = level.Info(logger).Log( "msg", "started cluster info retriever", "interval", (*esClusterInfoInterval).String(), ) case clusterinfo.ErrInitialCallTimeout: _ = level.Info(logger).Log("msg", "initial cluster info call timed out") default: _ = level.Error(logger).Log("msg", "failed to run cluster info retriever", "err", err) os.Exit(1) } // register cluster info retriever as prometheus collector prometheus.MustRegister(clusterInfoRetriever) mux := http.DefaultServeMux mux.Handle(*metricsPath, prometheus.Handler()) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, err = w.Write([]byte(` Elasticsearch Exporter

Elasticsearch Exporter

Metrics

`)) if err != nil { _ = level.Error(logger).Log( "msg", "failed handling writer", "err", err, ) } }) // health endpoint mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { http.Error(w, http.StatusText(http.StatusOK), http.StatusOK) }) server.Handler = mux server.Addr = *listenAddress _ = level.Info(logger).Log( "msg", "starting elasticsearch_exporter", "addr", *listenAddress, ) go func() { if err := server.ListenAndServe(); err != nil { _ = level.Error(logger).Log( "msg", "http server quit", "err", err, ) os.Exit(1) } }() c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) // create a context for graceful http server shutdown srvCtx, srvCancel := context.WithTimeout(context.Background(), 5*time.Second) defer srvCancel() <-c _ = level.Info(logger).Log("msg", "shutting down") _ = server.Shutdown(srvCtx) cancel() } prometheus-elasticsearch-exporter-1.1.0+ds/pkg/000077500000000000000000000000001362377146000215565ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/pkg/clusterinfo/000077500000000000000000000000001362377146000241135ustar00rootroot00000000000000prometheus-elasticsearch-exporter-1.1.0+ds/pkg/clusterinfo/clusterinfo.go000066400000000000000000000164701362377146000270070ustar00rootroot00000000000000package clusterinfo import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "path" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) const ( namespace = "elasticsearch" subsystem = "clusterinfo" ) var ( // ErrConsumerAlreadyRegistered is returned if a consumer is already registered ErrConsumerAlreadyRegistered = errors.New("consumer already registered") // ErrInitialCallTimeout is returned if the initial clusterinfo call timed out ErrInitialCallTimeout = errors.New("initial cluster info call timed out") initialTimeout = 10 * time.Second ) type consumer interface { // ClusterLabelUpdates returns a pointer to channel for cluster label updates ClusterLabelUpdates() *chan *Response // String implements the stringer interface String() string } // Retriever periodically gets the cluster info from the / endpoint end // sends it to all registered consumer channels type Retriever struct { consumerChannels map[string]*chan *Response logger log.Logger client *http.Client url *url.URL interval time.Duration sync chan struct{} versionMetric *prometheus.GaugeVec up *prometheus.GaugeVec lastUpstreamSuccessTs *prometheus.GaugeVec lastUpstreamErrorTs *prometheus.GaugeVec } // New creates a new Retriever func New(logger log.Logger, client *http.Client, u *url.URL, interval time.Duration) *Retriever { return &Retriever{ consumerChannels: make(map[string]*chan *Response), logger: logger, client: client, url: u, interval: interval, sync: make(chan struct{}, 1), versionMetric: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "version_info"), Help: "Constant metric with ES version information as labels", }, []string{ "cluster", "cluster_uuid", "build_date", "build_hash", "version", "lucene_version", }, ), up: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "up"), Help: "Up metric for the cluster info collector", }, []string{"url"}, ), lastUpstreamSuccessTs: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "last_retrieval_success_ts"), Help: "Timestamp of the last successful cluster info retrieval", }, []string{"url"}, ), lastUpstreamErrorTs: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, subsystem, "last_retrieval_failure_ts"), Help: "Timestamp of the last failed cluster info retrieval", }, []string{"url"}, ), } } // Describe implements the prometheus.Collector interface func (r *Retriever) Describe(ch chan<- *prometheus.Desc) { r.versionMetric.Describe(ch) r.up.Describe(ch) r.lastUpstreamSuccessTs.Describe(ch) r.lastUpstreamErrorTs.Describe(ch) } // Collect implements the prometheus.Collector interface func (r *Retriever) Collect(ch chan<- prometheus.Metric) { r.versionMetric.Collect(ch) r.up.Collect(ch) r.lastUpstreamSuccessTs.Collect(ch) r.lastUpstreamErrorTs.Collect(ch) } func (r *Retriever) updateMetrics(res *Response) { u := *r.url u.User = nil url := u.String() _ = level.Debug(r.logger).Log("msg", "updating cluster info metrics") // scrape failed, response is nil if res == nil { r.up.WithLabelValues(url).Set(0.0) r.lastUpstreamErrorTs.WithLabelValues(url).Set(float64(time.Now().Unix())) return } r.up.WithLabelValues(url).Set(1.0) r.versionMetric.WithLabelValues( res.ClusterName, res.ClusterUUID, res.Version.BuildDate, res.Version.BuildHash, res.Version.Number.String(), res.Version.LuceneVersion.String(), ) r.lastUpstreamSuccessTs.WithLabelValues(url).Set(float64(time.Now().Unix())) } // Update triggers an external cluster info label update func (r *Retriever) Update() { r.sync <- struct{}{} } // RegisterConsumer registers a consumer for cluster info updates func (r *Retriever) RegisterConsumer(c consumer) error { if _, registered := r.consumerChannels[c.String()]; registered { return ErrConsumerAlreadyRegistered } r.consumerChannels[c.String()] = c.ClusterLabelUpdates() return nil } // Run starts the update loop and periodically queries the / endpoint // The update loop is terminated upon ctx cancellation. The call blocks until the first // call to the cluster info endpoint was successful func (r *Retriever) Run(ctx context.Context) error { startupComplete := make(chan struct{}) // start update routine go func(ctx context.Context) { for { select { case <-ctx.Done(): _ = level.Info(r.logger).Log( "msg", "context cancelled, exiting cluster info update loop", "err", ctx.Err(), ) return case <-r.sync: _ = level.Info(r.logger).Log( "msg", "providing consumers with updated cluster info label", ) res, err := r.fetchAndDecodeClusterInfo() if err != nil { _ = level.Error(r.logger).Log( "msg", "failed to retrieve cluster info from ES", "err", err, ) r.updateMetrics(nil) continue } r.updateMetrics(res) for name, consumerCh := range r.consumerChannels { _ = level.Debug(r.logger).Log( "msg", "sending update", "consumer", name, "res", fmt.Sprintf("%+v", res), ) *consumerCh <- res } // close startupComplete if not already closed select { case <-startupComplete: default: close(startupComplete) } } } }(ctx) // trigger initial cluster info call _ = level.Info(r.logger).Log( "msg", "triggering initial cluster info call", ) r.sync <- struct{}{} // start a ticker routine go func(ctx context.Context) { if r.interval <= 0 { _ = level.Info(r.logger).Log( "msg", "no periodic cluster info label update requested", ) return } ticker := time.NewTicker(r.interval) for { select { case <-ctx.Done(): _ = level.Info(r.logger).Log( "msg", "context cancelled, exiting cluster info trigger loop", "err", ctx.Err(), ) return case <-ticker.C: _ = level.Debug(r.logger).Log( "msg", "triggering periodic update", ) r.sync <- struct{}{} } } }(ctx) // block until the first retrieval was successful select { case <-startupComplete: // first sync has been successful _ = level.Debug(r.logger).Log("msg", "initial clusterinfo sync succeeded") return nil case <-time.After(initialTimeout): // initial call timed out return ErrInitialCallTimeout case <-ctx.Done(): // context cancelled return nil } } func (r *Retriever) fetchAndDecodeClusterInfo() (*Response, error) { var response *Response u := *r.url u.Path = path.Join(r.url.Path, "/") res, err := r.client.Get(u.String()) if err != nil { _ = level.Error(r.logger).Log( "msg", "failed to get cluster info", "err", err, ) return nil, err } defer func() { err = res.Body.Close() if err != nil { _ = level.Warn(r.logger).Log( "msg", "failed to close http.Client", "err", err, ) } }() if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) } if err := json.NewDecoder(res.Body).Decode(&response); err != nil { return nil, err } return response, nil } prometheus-elasticsearch-exporter-1.1.0+ds/pkg/clusterinfo/clusterinfo_response.go000066400000000000000000000013421362377146000307150ustar00rootroot00000000000000package clusterinfo import ( "github.com/blang/semver" ) // Response is the cluster info retrievable from the / endpoint type Response struct { Name string `json:"name"` ClusterName string `json:"cluster_name"` ClusterUUID string `json:"cluster_uuid"` Version VersionInfo `json:"version"` Tagline string `json:"tagline"` } // VersionInfo is the version info retrievable from the / endpoint, embedded in Response type VersionInfo struct { Number semver.Version `json:"number"` BuildHash string `json:"build_hash"` BuildDate string `json:"build_date"` BuildSnapshot bool `json:"build_snapshot"` LuceneVersion semver.Version `json:"lucene_version"` } prometheus-elasticsearch-exporter-1.1.0+ds/pkg/clusterinfo/clusterinfo_test.go000066400000000000000000000106031362377146000300360ustar00rootroot00000000000000package clusterinfo import ( "context" "fmt" "net/http" "net/http/httptest" "net/url" "os" "reflect" "testing" "time" "github.com/go-kit/kit/log" "github.com/blang/semver" ) const ( nodeName = "test-node-" clusterName = "test-cluster-1" clusterUUID = "r1bT9sBrR7S9-CamE41Qqg" versionNumber = "5.6.9" buildHash = "877a590" buildDate = "2018-04-12T16:25:14.838Z" buildSnapshot = false luceneVersion = "6.6.1" tagline = "You Know, for Search" ) type mockES struct{} func (mockES) ServeHTTP(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, `{ "name" : "%s", "cluster_name" : "%s", "cluster_uuid" : "%s", "version" : { "number" : "%s", "build_hash" : "%s", "build_date" : "%s", "build_snapshot" : %t, "lucene_version" : "%s" }, "tagline" : "%s" }`, nodeName, clusterName, clusterUUID, versionNumber, buildHash, buildDate, buildSnapshot, luceneVersion, tagline, ) } type mockConsumer struct { name string data *Response ch chan *Response } func newMockConsumer(ctx context.Context, name string, t *testing.T) *mockConsumer { mc := &mockConsumer{ name: name, ch: make(chan *Response), } go func() { for { select { case d := <-mc.ch: mc.data = d t.Logf("consumer %s received data from channel: %+v\n", mc, mc.data) case <-ctx.Done(): t.Logf("shutting down consumer %s", mc) return } } }() return mc } func (mc *mockConsumer) String() string { return mc.name } func (mc *mockConsumer) ClusterLabelUpdates() *chan *Response { return &mc.ch } func TestNew(t *testing.T) { u, err := url.Parse("http://localhost:9200") if err != nil { t.Skipf("internal test error: %s", err) } r := New(log.NewNopLogger(), http.DefaultClient, u, 0) if r.url != u { t.Errorf("new Retriever mal-constructed") } } func TestRetriever_RegisterConsumer(t *testing.T) { mockES := httptest.NewServer(mockES{}) u, err := url.Parse(mockES.URL) if err != nil { t.Fatalf("internal test error: %s", err) } retriever := New(log.NewNopLogger(), mockES.Client(), u, 0) ctx, cancel := context.WithCancel(context.Background()) defer cancel() consumerNames := []string{"consumer-1", "consumer-2"} for _, n := range consumerNames { c := newMockConsumer(ctx, n, t) if err := retriever.RegisterConsumer(c); err != nil { t.Errorf("failed to register consumer: %s", err) } } if len(retriever.consumerChannels) != len(consumerNames) { t.Error("number of registered consumerChannels doesn't match the number of calls to the register func") } } func TestRetriever_fetchAndDecodeClusterInfo(t *testing.T) { // these override test package globals versionNumber, _ := semver.Make(versionNumber) luceneVersion, _ := semver.Make(luceneVersion) var expected = &Response{ Name: nodeName, ClusterName: clusterName, ClusterUUID: clusterUUID, Version: VersionInfo{ Number: versionNumber, BuildHash: buildHash, BuildDate: buildDate, BuildSnapshot: buildSnapshot, LuceneVersion: luceneVersion, }, Tagline: tagline, } mockES := httptest.NewServer(mockES{}) u, err := url.Parse(mockES.URL) if err != nil { t.Skipf("internal test error: %s", err) } retriever := New(log.NewNopLogger(), mockES.Client(), u, 0) ci, err := retriever.fetchAndDecodeClusterInfo() if err != nil { t.Fatalf("failed to retrieve cluster info: %s", err) } if !reflect.DeepEqual(ci, expected) { t.Errorf("unexpected response, want %v, got %v", expected, ci) } } func TestRetriever_Run(t *testing.T) { // setup mock ES mockES := httptest.NewServer(mockES{}) u, err := url.Parse(mockES.URL) if err != nil { t.Fatalf("internal test error: %s", err) } // setup cluster info retriever retriever := New(log.NewLogfmtLogger(os.Stdout), mockES.Client(), u, 0) // setup mock consumer ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() mc := newMockConsumer(ctx, "test-consumer", t) if err := retriever.RegisterConsumer(mc); err != nil { t.Fatalf("failed to register consumer: %s", err) } // start retriever retriever.Run(ctx) // trigger update retriever.Update() time.Sleep(20 * time.Millisecond) // ToDo: check mockConsumers received data t.Logf("%+v\n", mc.data) // check for deadlocks select { case <-ctx.Done(): if err := ctx.Err(); err == context.DeadlineExceeded { t.Fatal("context timeout exceeded, caught deadlock") } default: } } prometheus-elasticsearch-exporter-1.1.0+ds/tls.go000066400000000000000000000025431362377146000221320ustar00rootroot00000000000000package main import ( "crypto/tls" "crypto/x509" "io/ioutil" "log" ) func createTLSConfig(pemFile, pemCertFile, pemPrivateKeyFile string, insecureSkipVerify bool) *tls.Config { tlsConfig := tls.Config{} if insecureSkipVerify { // pem settings are irrelevant if we're skipping verification anyway tlsConfig.InsecureSkipVerify = true } if len(pemFile) > 0 { rootCerts, err := loadCertificatesFrom(pemFile) if err != nil { log.Fatalf("Couldn't load root certificate from %s. Got %s.", pemFile, err) return nil } tlsConfig.RootCAs = rootCerts } if len(pemCertFile) > 0 && len(pemPrivateKeyFile) > 0 { clientPrivateKey, err := loadPrivateKeyFrom(pemCertFile, pemPrivateKeyFile) if err != nil { log.Fatalf("Couldn't setup client authentication. Got %s.", err) return nil } tlsConfig.Certificates = []tls.Certificate{*clientPrivateKey} } return &tlsConfig } func loadCertificatesFrom(pemFile string) (*x509.CertPool, error) { caCert, err := ioutil.ReadFile(pemFile) if err != nil { return nil, err } certificates := x509.NewCertPool() certificates.AppendCertsFromPEM(caCert) return certificates, nil } func loadPrivateKeyFrom(pemCertFile, pemPrivateKeyFile string) (*tls.Certificate, error) { privateKey, err := tls.LoadX509KeyPair(pemCertFile, pemPrivateKeyFile) if err != nil { return nil, err } return &privateKey, nil }