.
ha_cluster_exporter-0.4.0/Makefile 0000664 0000000 0000000 00000004245 13574721033 0017246 0 ustar 00root root 0000000 0000000 # this is the what ends up in the RPM "Version" field and it is also used as suffix for the built binaries
# it can be arbitrary for local builds, but it if you want to commit to OBS it must correspond to a Git tag with an associated GitHub release
VERSION ?= dev
# we only use this to comply with RPM changelog conventions at SUSE
AUTHOR ?= shap-staff@suse.de
# you can customize any of the following to build forks
OBS_PROJECT ?= server:monitoring
OBS_PACKAGE ?= prometheus-ha_cluster_exporter
REPOSITORY ?= clusterlabs/ha_cluster_exporter
# the Go archs we crosscompile to
ARCHS ?= amd64 arm64 ppc64le s390x
default: clean mod-tidy fmt vet-check test build
download:
go mod download
go mod verify
build: amd64
build-all: clean-bin $(ARCHS)
$(ARCHS):
@mkdir -p build/bin
CGO_ENABLED=0 GOOS=linux GOARCH=$@ go build -trimpath -ldflags "-s -w -X main.version=$(VERSION)" -o build/bin/ha_cluster_exporter-$(VERSION)-$@
install:
go install
static-checks: vet-check fmt-check
vet-check: download
go vet .
fmt:
go fmt
mod-tidy:
go mod tidy
fmt-check:
.ci/go_lint.sh
test: download
go test -v
coverage: coverage.out
coverage.out:
go test -cover -coverprofile=coverage.out
go tool cover -html=coverage.out
clean: clean-bin clean-obs
go clean
rm -f coverage.out
clean-bin:
rm -rf build/bin
clean-obs:
rm -rf build/obs
obs-workdir: build/obs
build/obs:
@mkdir -p build/obs/$(OBS_PACKAGE)
osc checkout $(OBS_PROJECT)/$(OBS_PACKAGE) -o build/obs
cp ha_cluster_exporter.spec build/obs/$(OBS_PACKAGE).spec
sed -i 's/%%VERSION%%/$(VERSION)/' build/obs/$(OBS_PACKAGE).spec
git archive --format=tar HEAD | tar -x -C build/obs/$(OBS_PACKAGE)
cd build/obs/$(OBS_PACKAGE); go mod vendor
rm build/obs/*.tar.gz
tar -cvzf build/obs/$(OBS_PACKAGE)-$(VERSION).tar.gz -C build/obs/$(OBS_PACKAGE) .
.ci/gh_release_to_obs_changeset.py $(REPOSITORY) -a $(AUTHOR) -t $(VERSION) -f build/obs/$(OBS_PACKAGE).changes || true
obs-commit: obs-workdir
cd build/obs; osc addremove
cd build/obs; osc commit -m "Automated $(VERSION) release"
.PHONY: default download install static-checks vet-check fmt fmt-check mod-tidy test clean clean-bin clean-obs build build-all obs-commit obs-workdir $(ARCHS)
ha_cluster_exporter-0.4.0/README.md 0000664 0000000 0000000 00000011007 13574721033 0017057 0 ustar 00root root 0000000 0000000 # ha_cluster_exporter
[](https://travis-ci.org/ClusterLabs/ha_cluster_exporter)
This is a bespoke Prometheus exporter used to enable the monitoring of Pacemaker based HA clusters.
## Table of Contents
1. [Features](#features)
2. [Installation](#installation)
3. [Usage](#usage)
4. [Development](#development)
## Features
The exporter is a stateless HTTP endpoint. On each HTTP request, it locally inspects the cluster status by parsing pre-existing distributed data, provided by the tools of the various cluster components.
Exported data include:
- Pacemaker cluster summary, nodes and resources stats
- Corosync ring errors and quorum votes
- SBD devices health status
- DRBD resources and connections stats
A comprehensive list of all the metrics can be found in the [metrics document](doc/metric_spec.md).
## Installation
The project can be installed in many ways, including but not limited to:
1. [Manual clone & build](#manual-clone-&-build)
2. [Go](#go)
3. [RPM](#rpm)
### Manual clone & build
```
git clone https://github.com/ClusterLabs/ha_cluster_exporter
cd ha_cluster_exporter
make
make install
```
### Go
```
go get github.com/ClusterLabs/ha_cluster_exporter
```
### RPM
You can find the repositories for RPM based distributions in [SUSE's Open Build Service](https://build.opensuse.org/repositories/server:monitoring/prometheus-ha_cluster_exporter).
On openSUSE or SUSE Linux Enterprise you can just use the `zypper` system package manager:
```shell
export DISTRO=SLE_15_SP1 # change as desired
zypper addrepo https://download.opensuse.org/repositories/server:/monitoring/$DISTRO/server:monitoring.repo
zypper install prometheus-ha_cluster_exporter
```
## Usage
You can run the exporter in any of the cluster nodes.
```
$ ./ha_cluster_exporter
INFO[0000] Serving metrics on 0.0.0.0:9002
```
Though not strictly required, it is _strongly_ advised to run it in all the nodes.
It will export the metrics under the `/metrics` path, on port `9002` by default.
While the exporter can run outside a HA cluster node, it won't export any metric it can't collect; e.g. it won't export DRBD metrics if it can't be locally inspected with `drbdsetup`.
A warning message will inform the user of such cases.
**Hint:**
You can deploy a full HA Cluster via Terraform with [SUSE/ha-sap-terraform-deployments](https://github.com/SUSE/ha-sap-terraform-deployments).
### Configuration
All the runtime parameters can be configured either via CLI flags or via a configuration file, both or which are completely optional.
For more details, refer to the help message via `ha_cluster_exporter --help`.
**Note**:
the built-in defaults are tailored for the latest version of SUSE Linux Enterprise and openSUSE.
The program will scan, in order, the current working directory, `$HOME/.config`, `/etc` and `/usr/etc` for files named `ha_cluster_exporter.(yaml|json|toml)`.
The first match has precedence, and the CLI flags have precedence over the config file.
Please refer to the example [YAML configuration](ha_cluster_exporter.yaml) for more details.
### systemd integration
A [systemd unit file](ha_cluster_exporter.service) is provided with the RPM packages. You can enable and start it as usual:
```
systemctl --now enable prometheus-ha_cluster_exporter
```
## Development
Pull requests are more than welcome!
We recommend having a look at the [design document](doc/design.md) before contributing.
#### Makefile
Most development tasks can be accomplished via [make](Makefile).
The default target will clean, analyse, test and build the amd64 binary into the `build/bin` directory.
You can also cross-compile to the various architectures we support with `make build-all`.
##### Open Build Service releases
The CI will automatically publish GitHub releases to SUSE's Open Build Service: to perform a new release, just publish a new GH release or push a git tag. Tags must always follow the [SemVer](https://semver.org/) scheme.
If you wish to produce an OBS working directory locally, after you have configured [`osc`](https://en.opensuse.org/openSUSE:OSC) locally, you can run:
```
make obs-workdir
```
This will checkout the OBS project and prepare a release in the `build/obs` directory.
Note that, by default, `dev` is used as the RPM `Version` field, as well as a suffix for all the binary file names.
To prepare an actual release, you can use the `VERSION` environment variable to set this value to an actual release tag.
To commit the release to OBS, run `make obs-commit`.
ha_cluster_exporter-0.4.0/corosync_metrics.go 0000664 0000000 0000000 00000012651 13574721033 0021522 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
var (
corosyncMetrics = metricDescriptors{
// the map key will function as an identifier of the metric throughout the rest of the code;
// it is arbitrary, but by convention we use the actual metric name
"quorate": NewMetricDesc("corosync", "quorate", "Whether or not the cluster is quorate", nil),
"ring_errors_total": NewMetricDesc("corosync", "ring_errors_total", "Total number of corosync ring errors", nil),
"quorum_votes": NewMetricDesc("corosync", "quorum_votes", "Cluster quorum votes; one line per type", []string{"type"}),
}
)
func NewCorosyncCollector(cfgToolPath string, quorumToolPath string) (*corosyncCollector, error) {
err := CheckExecutables(cfgToolPath, quorumToolPath)
if err != nil {
return nil, errors.Wrap(err, "could not initialize Corosync collector")
}
return &corosyncCollector{
DefaultCollector{
metrics: corosyncMetrics,
},
cfgToolPath,
quorumToolPath,
}, nil
}
type corosyncCollector struct {
DefaultCollector
cfgToolPath string
quorumToolPath string
}
func (c *corosyncCollector) Collect(ch chan<- prometheus.Metric) {
c.mutex.Lock()
defer c.mutex.Unlock()
log.Infoln("Collecting corosync metrics...")
err := c.collectRingErrorsTotal(ch)
if err != nil {
log.Warnln(err)
}
quorumStatusRaw := c.getQuoromStatus()
quorumStatus, quorate, err := parseQuoromStatus(quorumStatusRaw)
if err != nil {
log.Warnln(err)
return
}
ch <- c.makeGaugeMetric("quorate", quorate)
for voteType, value := range quorumStatus {
ch <- c.makeGaugeMetric("quorum_votes", float64(value), voteType)
}
}
func (c *corosyncCollector) collectRingErrorsTotal(ch chan<- prometheus.Metric) error {
ringStatus := c.getCorosyncRingStatus()
ringErrorsTotal, err := parseRingStatus(ringStatus)
if err != nil {
return errors.Wrap(err, "cannot parse ring status")
}
ch <- c.makeGaugeMetric("ring_errors_total", float64(ringErrorsTotal))
return nil
}
func (c *corosyncCollector) getQuoromStatus() []byte {
// We suppress the exec error because if any interface is faulty, the tool will exit with code 1.
// If all interfaces are active, exit code will be 0.
quorumInfoRaw, _ := exec.Command(c.quorumToolPath).Output()
return quorumInfoRaw
}
func parseQuoromStatus(quoromStatusRaw []byte) (quorumVotes map[string]int, quorate float64, err error) {
quoromRaw := string(quoromStatusRaw)
// Quorate: Yes
// Votequorum information
// ----------------------
// Expected votes: 2
// Highest expected: 2
// Total votes: 2
// Quorum: 1
// We apply the same method for all the metrics/data:
// first split the string for finding the word , e.g "Expected votes:", and get it via regex
// only the number 2,
// and convert it to integer type
numberOnly := regexp.MustCompile("[0-9]+")
wordOnly := regexp.MustCompile("[a-zA-Z]+")
quoratePresent := regexp.MustCompile("Quorate:")
// In case of error, the binary is there but execution was erroring out, check output for quorate string.
quorateWordPresent := quoratePresent.FindString(string(quoromRaw))
// check the case there is an sbd_config but the SBD_DEVICE is not set
if quorateWordPresent == "" {
return nil, quorate, fmt.Errorf("cannot parse quorum status")
}
quorateRaw := wordOnly.FindString(strings.SplitAfterN(quoromRaw, "Quorate:", 2)[1])
quorateString := strings.ToLower(quorateRaw)
if quorateString == "yes" {
quorate = 1
}
expVotes, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Expected votes:", 2)[1]))
highVotes, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Highest expected:", 2)[1]))
totalVotes, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Total votes:", 2)[1]))
quorum, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Quorum:", 2)[1]))
quorumVotes = map[string]int{
"expected_votes": expVotes,
"highest_expected": highVotes,
"total_votes": totalVotes,
"quorum": quorum,
}
if len(quorumVotes) == 0 {
return quorumVotes, quorate, fmt.Errorf("could not retrieve any quorum information")
}
return quorumVotes, quorate, nil
}
// get status ring and return it as bytes
// this function can return also just an malformed output in case of error, we don't check.
// It is the parser that will check the status
func (c *corosyncCollector) getCorosyncRingStatus() []byte {
// We suppress the exec error because if any interface is faulty, the tool will exit with code 1.
// If all interfaces are active/without error, exit code will be 0.
ringStatusRaw, _ := exec.Command(c.cfgToolPath, "-s").Output()
return ringStatusRaw
}
// return the number of RingError that we will use as gauge, and error if somethings unexpected happens
func parseRingStatus(ringStatus []byte) (int, error) {
statusRaw := string(ringStatus)
// check if there is a ring ERROR first
ringErrorsTotal := strings.Count(statusRaw, "FAULTY")
// in case there is no error we need to check that the output is not
if ringErrorsTotal == 0 {
// if there is no RING ID word, the command corosync-cfgtool went wrong/error out
if strings.Count(statusRaw, "RING ID") == 0 {
return 0, fmt.Errorf("corosync-cfgtool returned unexpected output: %s", statusRaw)
}
return 0, nil
}
// there is a ringError
return ringErrorsTotal, nil
}
ha_cluster_exporter-0.4.0/doc/ 0000775 0000000 0000000 00000000000 13574721033 0016346 5 ustar 00root root 0000000 0000000 ha_cluster_exporter-0.4.0/doc/design.md 0000664 0000000 0000000 00000002743 13574721033 0020147 0 ustar 00root root 0000000 0000000 # Design Notes
This document describes the rationale behind design decisions takend during the development of this project.
## Goals
- Export runtime statistics about the various HA cluster components from existing data sources, to be consumed in a Prometheus monitoring stack.
## Non-goals
- Maintain an internal, consistent, persisting representation of the cluster state; since the original source of truth is distributed, we want to avoid the complexity of a stateful middleware.
## Structure
The project consist in a small HTTP application that exposes runtime data in a line protocol.
A series of `prometheus.Collector` implementations, one for each cluster component (that we'll call _subsystems_) are instantiated in the main application entry point, registered with the Prometheus client, and then exposed via its HTTP handler.
Each collector `Collect` method will be called concurrently by the client itself in an internal worker goroutine.
The sources hence are read every time an HTTP request comes, and the collected data is not shared: its lifecycle corresponds with the request's.
To avoid concurrent reads of the same source, all `Collect` methods are serialized with a mutex.
## Collectors
The collectors are very simple: they usually just invoke a bunch of system commands, then parse the output into bespoke data structures that can be used to build Prometheus metrics.
More details about these metrics can be found in the [metrics specification document](metric_spec.md).
ha_cluster_exporter-0.4.0/doc/metric_spec.md 0000664 0000000 0000000 00000033575 13574721033 0021202 0 ustar 00root root 0000000 0000000 # Metrics specification
This document describes the metrics exposed by `ha_cluster_exporter`.
General notes:
- All the metrics are _namespaced_ with the prefix `ha_cluster`, which is followed by a _subsystem_, and both are in turn composed into a _Fully Qualified Name_ (FQN) of each metrics.
- All the metrics and labels _names_ are in snake_case, as conventional with Prometheus. That said, as much as we'll try to keep this consistent throughout the project, the label _values_ may not actually follow this convention, though (e.g. value is a hostname).
- All the metrics are timestamped with the Unix epoch time in milliseconds; in the provided examples, this value will always be `1234`.
- Some metrics, like `ha_cluster_pacemaker_nodes`, `ha_cluster_pacemaker_resources`, share common traits:
- their labels contain the relevant data you may want to track or use for aggregation and filtering;
- either their value is `1`, or the line is absent altogether; this is because each line represents one entity of the cluster, but the exporter itself is stateless, i.e. we don't track the life-cycle of entities that do not exist anymore in the cluster.
These are the currently implemented subsystems.
1. [Pacemaker](#pacemaker)
2. [Corosync](#corosync)
3. [SBD](#sbd)
4. [DRBD](#drbd)
## Pacemaker
The Pacemaker subsystem collects an atomic snapshot of the HA cluster directly from the XML CIB of Pacemaker via `crm_mon`.
0. [Sample](../test/pacemaker.metrics)
1. [`ha_cluster_pacemaker_config_last_change`](#ha_cluster_pacemaker_config_last_change)
3. [`ha_cluster_pacemaker_fail_count`](#ha_cluster_pacemaker_fail_count)
2. [`ha_cluster_pacemaker_location_constraints`](#ha_cluster_pacemaker_location_constraints)
4. [`ha_cluster_pacemaker_migration_threshold`](#ha_cluster_pacemaker_migration_threshold)
5. [`ha_cluster_pacemaker_nodes_total`](#ha_cluster_pacemaker_nodes_total)
6. [`ha_cluster_pacemaker_nodes`](#ha_cluster_pacemaker_nodes)
7. [`ha_cluster_pacemaker_resources_total`](#ha_cluster_pacemaker_resources_total)
8. [`ha_cluster_pacemaker_resources`](#ha_cluster_pacemaker_resources)
9. [`ha_cluster_pacemaker_stonith_enabled`](#ha_cluster_pacemaker_stonith_enabled)
### `ha_cluster_pacemaker_config_last_change`
#### Description
The value of this metric is a Unix timestamp in seconds, converted to a float, corresponding to the last time Pacemaker configuration changed.
The metric is in turn timestamped with the time it was last checked.
### `ha_cluster_pacemaker_fail_count`
#### Description
The number of fail count per node and resource ID.
The value is an integer ranging from 0 to `+Inf`.
The actual maximum integer value depends on Pacemaker internals, so please refer to upstream documentation for further information.
### `ha_cluster_pacemaker_location_constraints`
#### Description
Resource location constraints.
The value of the metric is the **score** of the constraint, represented by an integer ranging from `-Inf` to `+Inf`.
The actual minimum and maximum integer values depend on Pacemaker internals, so please refer to upstream documentation for further information.
#### Labels
- `constraint`: the unique string identifier of the constraint.
- `node`: the node the constraint applies to.
- `resource`: the resource the constraint applies to.
- `role`: the resource role the constraint applies to, if any.
### `ha_cluster_pacemaker_migration_threshold`
#### Description
The number of migration threshold pro node and resource ID set by a pacemaker cluster.
Possible values are positive numbers.
### `ha_cluster_pacemaker_nodes`
#### Description
The nodes in the cluster; one line per `node`, per `status`.
Either the value is `1`, or the line is absent altogether.
#### Labels
- `node`: name of the node (usually the hostname).
- `status`: one of `online|standby|standby_onfail|maintanance|pending|unclean|shutdown|expected_up|dc`.
- `type`: one of `member|ping|remote`.
The total number of lines for this metric will be the cardinality of `name` times the cardinality of `status`.
### `ha_cluster_pacemaker_nodes_total`
#### Description
The total number of *configured* nodes in the cluster. This value is mostly static and *does not* take into account the status of the nodes. It only changes when the Pacemaker configuration changes.
### `ha_cluster_pacemaker_resources`
#### Description
The resources in the cluster; one line per `id`, per `status`.
Either the value is `1`, or the line is absent altogether.
#### Labels
- `resource`: the unique resource name.
- `node`: the name of the node hosting the resource.
- `managed`: either `true` or `false`.
- `role`: one of `started|stopped|master|slave` or one of `starting|stopping|migrating|promoting|demoting`.
- `status` one of `active|orphaned|blocked|failed|failure_ignored`.
The total number of lines for this metric will be the cardinality of `id` times the cardinality of `status`.
### `ha_cluster_pacemaker_resources_total`
#### Description
The total number of *configured* resources in the cluster. This value is mostly static and *does not* take into account the status of the resources. It only changes when the Pacemaker configuration changes.
### `ha_cluster_pacemaker_stonith_enabled`
#### Description
Whether or not stonith is enabled in the cluster.
Value is either `1` or `0`.
## Corosync
The Corosync subsystem collects cluster quorum votes and ring status by parsing the output of `corosync-quorumtool` and `corosync-cfgtool`.
0. [Sample](../test/corosync.metrics)
1. [`ha_cluster_corosync_quorate`](#ha_cluster_corosync_quorate)
2. [`ha_cluster_corosync_quorum_votes`](#ha_cluster_corosync_quorum_votes)
3. [`ha_cluster_corosync_ring_errors_total`](#ha_cluster_corosync_ring_errors_total)
### `ha_cluster_corosync_quorate`
#### Description
Whether or not the cluster is quorate.
Value is either `1` or `0`.
### `ha_cluster_corosync_quorum_votes`
#### Description
Cluster quorum votes; one line per type.
#### Labels
- `type`: one of `expected_votes|highest_expected|total_votes|quorum`
### `ha_cluster_corosync_ring_errors_total`
#### Description
Total number of corosync ring errors.
## SBD
The SBD subsystems collect devices stats by parsing its configuration the output of `sbd --dump`.
0. [Sample](../test/sbd.metrics)
1. [`ha_cluster_sbd_device_status`](#ha_cluster_sbd_device_status)
2. [`ha_cluster_sbd_devices_total`](#ha_cluster_sbd_devices_total)
### `ha_cluster_sbd_device_status`
#### Description
Whether or not an SBD device is healthy. One line per `device`.
Value is either `1` or `0`.
#### Labels
- `device`: the path of the device.
The total number of lines for this metric will be the cardinality of `device`.
### `ha_cluster_sbd_devices_total`
#### Description
Total count of configured SBD devices.
Value is an integer greater than or equal to `0`.
## DRBD
The DRBD subsystems collect devices stats by parsing its configuration the JSON output of `drbdsetup`.
0. [Sample](../test/drbd.metrics)
1. [`ha_cluster_drbd_resources`](#ha_cluster_drbd_resources)
2. [`ha_cluster_drbd_written`](#ha_cluster_drbd_written)
3. [`ha_cluster_drbd_read`](#ha_cluster_drbd_read)
4. [`ha_cluster_drbd_al_writes`](#ha_cluster_al_writes)
5. [`ha_cluster_drbd_bm_writes`](#ha_cluster_bm_writes)
6. [`ha_cluster_drbd_upper_pending`](#ha_cluster_drbd_upper_pending)
7. [`ha_cluster_drbd_lower_pending`](#ha_cluster_drbd_lower_pending)
8. [`ha_cluster_drbd_quorum`](#ha_cluster_drbd_quorum)
9. [`ha_cluster_drbd_connections`](#ha_cluster_drbd_connections)
10. [`ha_cluster_drbd_connections_sync`](#ha_cluster_drbd_connections_sync)
11. [`ha_cluster_drbd_connections_received`](#ha_cluster_drbd_connections_received)
12. [`ha_cluster_drbd_connections_sent`](#ha_cluster_drbd_connections_sent)
13. [`ha_cluster_drbd_connections_pending`](#ha_cluster_drbd_connections_pending)
14. [`ha_cluster_drbd_connections_unacked`](#ha_cluster_drbd_connections_unacked)
15. [`ha_cluster_drbd_split_brain`](#ha_cluster_drbd_split_brain)
### `ha_cluster_drbd_connections`
#### Description
The DRBD resource connections; 1 line per per `resource`, per `peer_node_id`
Either the value is `1`, or the line is absent altogether.
#### Labels
- `resource`: the resource this connection is for.
- `peer_node_id`: the id of the node this connection is for
- `peer_role`: one of `primary|secondary|unknown`
- `volume`: the volume number
- `peer_disk_state`: one of `attaching|failed|negotiating|inconsistent|outdated|dunknown|consistent|uptodate`
The total number of lines for this metric will be the cardinality of `resource` times the cardinality of `peer_node_id`.
### `ha_cluster_drbd_connections_sync`
#### Description
The DRBD disk connections in sync percentage. Values are float from `0` to `100.00`.
#### Labels
- `resource`: the resource this connection is for.
- `peer_node_id`: the id of the node this connection is for
- `volume`: the volume number
### `ha_cluster_drbd_connections_received`
#### Description
Volume of net data received from the partner via the network connection in KiB; 1 line per per `resource`, per `peer_node_id`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the resource this connection is for.
- `peer_node_id`: the id of the node this connection is for
- `volume`: the volume number
### `ha_cluster_drbd_connections_sent`
#### Description
Volume of net data sent to the partner via the network connection in KiB; 1 line per per `resource`, per `peer_node_id`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the resource this connection is for.
- `peer_node_id`: the id of the node this connection is for
- `volume`: the volume number
### `ha_cluster_drbd_connections_pending`
#### Description
Number of requests sent to the partner that have not yet been received; 1 line per per `resource`, per `peer_node_id`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the resource this connection is for.
- `peer_node_id`: the id of the node this connection is for
- `volume`: the volume number
### `ha_cluster_drbd_connections_unacked`
#### Description
Number of requests received by the partner but have not yet been acknowledged; 1 line per per `resource`, per `peer_node_id`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the resource this connection is for.
- `peer_node_id`: the id of the node this connection is for
- `volume`: the volume number
### `ha_cluster_drbd_resources`
#### Description
The DRBD resources; 1 line per `name`, per `volume`
Either the value is `1`, or the line is absent altogether.
#### Labels
- `resource`: the name of the resource.
- `role`: one of `primary|secondary|unknown`
- `volume`: the volume number
- `disk_state`: one of `attaching|failed|negotiating|inconsistent|outdated|dunknown|consistent|uptodate`
The total number of lines for this metric will be the cardinality of `name` times the cardinality of `volume`.
### `ha_cluster_drbd_written`
#### Description
Amount in KiB written to the DRBD resource; 1 line per `resource`, per `volume`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_read`
#### Description
Amount in KiB read from the DRBD resource; 1 line per `resource`, per `volume`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_al_writes`
#### Description
Number of updates of the activity log area of the meta data; 1 line per `resource`, per `volume`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_bm_writes`
#### Description
Number of updates of the bitmap area of the meta data; 1 line per `resource`, per `volume`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_upper_pending`
#### Description
Number of block I/O requests forwarded to DRBD, but not yet answered by DRBD; 1 line per `resource`, per `volume`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_lower_pending`
#### Description
Number of open requests to the local I/O sub-system issued by DRBD; 1 line per `resource`, per `volume`
Value is an integer greater than or equal to `0`.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_quorum`
#### Description
Quorum status of the DRBD resource according to it's configured quorum policies; 1 line per `resource`, per `volume`
Value is `1` when quorate, or `0` when inquorate.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
### `ha_cluster_drbd_split_brain`
#### Description
This metric signal if there is a split brain occurring per resource and volume.
Either the value is `1`, or the line is absent altogether.
This metric is a special metric compared to others, because in order to make this metric work you will need to setup a DRBD custom split-brain handler. Look at the end.
#### Labels
- `resource`: the name of the resource.
- `volume`: the volume number
#### Setting up the DRBD split-brain hook
In order to get the `split_brain` metric working:
1) copy hook into all drbd nodes:
get the hook from:
https://github.com/SUSE/ha-sap-terraform-deployments/blob/72c9d3ecf6c3f6dd18ccb7bcbde4b40722d5c641/salt/drbd_node/files/notify-split-brain-haclusterexporter-suse-metric.sh
2) on the DRBD configuration enable the hook:
```split_brain: "/usr/lib/drbd/notify-split-brain-haclusterexporter-suse-metric.sh"```
Refer to upstream doc: https://docs.linbit.com/docs/users-guide-8.4/#s-configure-split-brain-behavior
It is important for the exporter that he hook should create the files in that location and naming.
Remember to remove the files manually after the split brain is solved
ha_cluster_exporter-0.4.0/drbd_metrics.go 0000664 0000000 0000000 00000020647 13574721033 0020602 0 ustar 00root root 0000000 0000000 package main
import (
"encoding/json"
"io/ioutil"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
// drbdStatus is for parsing relevant data we want to convert to metrics
type drbdStatus struct {
Name string `json:"name"`
Role string `json:"role"`
Devices []struct {
Volume int `json:"volume"`
Written int `json:"written"`
Read int `json:"read"`
AlWrites int `json:"al-writes"`
BmWrites int `json:"bm-writes"`
UpPending int `json:"upper-pending"`
LoPending int `json:"lower-pending"`
Quorum bool `json:"quorum"`
DiskState string `json:"disk-state"`
} `json:"devices"`
Connections []struct {
PeerNodeID int `json:"peer-node-id"`
PeerRole string `json:"peer-role"`
PeerDevices []struct {
Volume int `json:"volume"`
Received int `json:"received"`
Sent int `json:"sent"`
Pending int `json:"pending"`
Unacked int `json:"unacked"`
PeerDiskState string `json:"peer-disk-state"`
PercentInSync float64 `json:"percent-in-sync"`
} `json:"peer_devices"`
} `json:"connections"`
}
var (
drbdMetrics = metricDescriptors{
// the map key will function as an identifier of the metric throughout the rest of the code;
// it is arbitrary, but by convention we use the actual metric name
"resources": NewMetricDesc("drbd", "resources", "The DRBD resources; 1 line per name, per volume", []string{"resource", "role", "volume", "disk_state"}),
"written": NewMetricDesc("drbd", "written", "KiB written to DRBD; 1 line per res, per volume", []string{"resource", "volume"}),
"read": NewMetricDesc("drbd", "read", "KiB read from DRBD; 1 line per res, per volume", []string{"resource", "volume"}),
"al_writes": NewMetricDesc("drbd", "al_writes", "Writes to activity log; 1 line per res, per volume", []string{"resource", "volume"}),
"bm_writes": NewMetricDesc("drbd", "bm_writes", "Writes to bitmap; 1 line per res, per volume", []string{"resource", "volume"}),
"upper_pending": NewMetricDesc("drbd", "upper_pending", "Upper pending; 1 line per res, per volume", []string{"resource", "volume"}),
"lower_pending": NewMetricDesc("drbd", "lower_pending", "Lower pending; 1 line per res, per volume", []string{"resource", "volume"}),
"quorum": NewMetricDesc("drbd", "quorum", "Quorum status per resource and per volume", []string{"resource", "volume"}),
"connections": NewMetricDesc("drbd", "connections", "The DRBD resource connections; 1 line per per resource, per peer_node_id", []string{"resource", "peer_node_id", "peer_role", "volume", "peer_disk_state"}),
"connections_sync": NewMetricDesc("drbd", "connections_sync", "The in sync percentage value for DRBD resource connections", []string{"resource", "peer_node_id", "volume"}),
"connections_received": NewMetricDesc("drbd", "connections_received", "KiB received per connection", []string{"resource", "peer_node_id", "volume"}),
"connections_sent": NewMetricDesc("drbd", "connections_sent", "KiB sent per connection", []string{"resource", "peer_node_id", "volume"}),
"connections_pending": NewMetricDesc("drbd", "connections_pending", "Pending value per connection", []string{"resource", "peer_node_id", "volume"}),
"connections_unacked": NewMetricDesc("drbd", "connections_unacked", "Unacked value per connection", []string{"resource", "peer_node_id", "volume"}),
"split_brain": NewMetricDesc("drbd", "split_brain", "Whether a split brain has been detected; 1 line per resource, per volume.", []string{"resource", "volume"}),
}
)
func NewDrbdCollector(drbdSetupPath string, drbdSplitBrainPath string) (*drbdCollector, error) {
err := CheckExecutables(drbdSetupPath)
if err != nil {
return nil, errors.Wrap(err, "could not initialize DRBD collector")
}
return &drbdCollector{
DefaultCollector{
metrics: drbdMetrics,
},
drbdSetupPath,
drbdSplitBrainPath,
}, nil
}
type drbdCollector struct {
DefaultCollector
drbdsetupPath string
drbdSplitBrainPath string
}
func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) {
c.mutex.Lock()
defer c.mutex.Unlock()
log.Infoln("Collecting DRBD metrics...")
// set split brain metric
c.setDrbdSplitBrainMetric(ch)
drbdStatusRaw, err := exec.Command(c.drbdsetupPath, "status", "--json").Output()
if err != nil {
log.Warnf("Error while retrieving drbd infos %s", err)
return
}
// populate structs and parse relevant info we will expose via metrics
drbdDev, err := parseDrbdStatus(drbdStatusRaw)
if err != nil {
log.Warnf("Error while parsing drbd json: %s", err)
return
}
for _, resource := range drbdDev {
for _, device := range resource.Devices {
// the `resources` metric value is always 1, otherwise it's absent
ch <- c.makeGaugeMetric("resources", float64(1), resource.Name, resource.Role, strconv.Itoa(device.Volume), strings.ToLower(device.DiskState))
ch <- c.makeGaugeMetric("written", float64(device.Written), resource.Name, strconv.Itoa(device.Volume))
ch <- c.makeGaugeMetric("read", float64(device.Read), resource.Name, strconv.Itoa(device.Volume))
ch <- c.makeGaugeMetric("al_writes", float64(device.AlWrites), resource.Name, strconv.Itoa(device.Volume))
ch <- c.makeGaugeMetric("bm_writes", float64(device.BmWrites), resource.Name, strconv.Itoa(device.Volume))
ch <- c.makeGaugeMetric("upper_pending", float64(device.UpPending), resource.Name, strconv.Itoa(device.Volume))
ch <- c.makeGaugeMetric("lower_pending", float64(device.LoPending), resource.Name, strconv.Itoa(device.Volume))
if bool(device.Quorum) == true {
ch <- c.makeGaugeMetric("quorum", float64(1), resource.Name, strconv.Itoa(device.Volume))
} else {
ch <- c.makeGaugeMetric("quorum", float64(0), resource.Name, strconv.Itoa(device.Volume))
}
}
if len(resource.Connections) == 0 {
log.Warnf("Could not retrieve connection info for resource '%s'\n", resource.Name)
continue
}
// a Resource can have multiple connection with different nodes
for _, conn := range resource.Connections {
if len(conn.PeerDevices) == 0 {
log.Warnf("Could not retrieve any peer device info for connection '%d'\n", conn.PeerNodeID)
continue
}
for _, peerDev := range conn.PeerDevices {
ch <- c.makeGaugeMetric("connections", float64(1), resource.Name, strconv.Itoa(conn.PeerNodeID),
conn.PeerRole, strconv.Itoa(peerDev.Volume), strings.ToLower(peerDev.PeerDiskState))
ch <- c.makeGaugeMetric("connections_sync", float64(peerDev.PercentInSync), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
ch <- c.makeGaugeMetric("connections_received", float64(peerDev.Received), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
ch <- c.makeGaugeMetric("connections_sent", float64(peerDev.Sent), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
ch <- c.makeGaugeMetric("connections_pending", float64(peerDev.Pending), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
ch <- c.makeGaugeMetric("connections_unacked", float64(peerDev.Unacked), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
}
}
}
}
func parseDrbdStatus(statusRaw []byte) ([]drbdStatus, error) {
var drbdDevs []drbdStatus
err := json.Unmarshal(statusRaw, &drbdDevs)
if err != nil {
return drbdDevs, err
}
return drbdDevs, nil
}
func (c *drbdCollector) setDrbdSplitBrainMetric(ch chan<- prometheus.Metric) {
// set split brain metric
// by default if the custom hook is not set, the exporter will not be able to detect it
files, _ := ioutil.ReadDir(c.drbdSplitBrainPath)
// the split brain files exists
for _, f := range files {
// check if in directory there are file of syntax we expect (nil is when there is not any)
match, _ := filepath.Glob(c.drbdSplitBrainPath + "/drbd-split-brain-detected-*")
if match == nil {
continue
}
resAndVolume := strings.Split(f.Name(), "drbd-split-brain-detected-")[1]
// avoid to have index out range panic error (in case the there is not resource-volume syntax)
if len(strings.Split(resAndVolume, "-")) != 2 {
continue
}
//Resource (0) volume (1) place in slice
resourceAndVolume := strings.Split(resAndVolume, "-")
ch <- c.makeGaugeMetric("split_brain", float64(1), resourceAndVolume[0], resourceAndVolume[1])
}
}
ha_cluster_exporter-0.4.0/go.mod 0000664 0000000 0000000 00000000363 13574721033 0016711 0 ustar 00root root 0000000 0000000 module github.com/ClusterLabs/ha_cluster_exporter
go 1.12
require (
github.com/pkg/errors v0.8.0
github.com/prometheus/client_golang v1.1.0
github.com/sirupsen/logrus v1.4.2
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.5.0
)
ha_cluster_exporter-0.4.0/go.sum 0000664 0000000 0000000 00000040502 13574721033 0016735 0 ustar 00root root 0000000 0000000 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4=
github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
ha_cluster_exporter-0.4.0/ha_cluster_exporter.go 0000664 0000000 0000000 00000013470 13574721033 0022216 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"net/http"
"os"
"sync"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
config "github.com/spf13/viper"
)
const NAMESPACE = "ha_cluster"
type Clock interface {
Now() time.Time
}
type SystemClock struct{}
func (SystemClock) Now() time.Time {
return time.Now()
}
type metricDescriptors map[string]*prometheus.Desc
type DefaultCollector struct {
metrics metricDescriptors
mutex sync.RWMutex
}
func (c *DefaultCollector) Describe(ch chan<- *prometheus.Desc) {
for _, metric := range c.metrics {
ch <- metric
}
}
func (c *DefaultCollector) makeGaugeMetric(metricKey string, value float64, labelValues ...string) prometheus.Metric {
desc, ok := c.metrics[metricKey]
if !ok {
// we hard panic on this because it's most certainly a coding error
panic(errors.Errorf("undeclared metric '%s'", metricKey))
}
return prometheus.NewMetricWithTimestamp(clock.Now(), prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labelValues...))
}
// Convenience wrapper around Prometheus constructors.
// Produces a metric with name `NAMESPACE_subsystem_name`.
// `NAMESPACE` is a global project constant;
// `subsystem` is an arbitrary name used to group related metrics under the same name prefix;
// `name` is the last and most relevant part of the metrics Full Qualified Name;
// `variableLabels` is a list of labels to declare. Use `nil` to declare no labels.
func NewMetricDesc(subsystem, name, help string, variableLabels []string) *prometheus.Desc {
return prometheus.NewDesc(prometheus.BuildFQName(NAMESPACE, subsystem, name), help, variableLabels, nil)
}
// check that all the given paths exist and are executable files
func CheckExecutables(paths ...string) error {
for _, path := range paths {
fileInfo, err := os.Stat(path)
if err != nil || os.IsNotExist(err) {
return errors.Errorf("'%s' does not exist", path)
}
if fileInfo.IsDir() {
return errors.Errorf("'%s' is a directory", path)
}
if (fileInfo.Mode() & 0111) == 0 {
return errors.Errorf("'%s' is not executable", path)
}
}
return nil
}
// Landing Page (for /)
func landingpage(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`
HACluster Exporter
HACluster Exporter
Metrics
More information:
github.com/ClusterLabs/ha_cluster_exporter
`))
}
func loglevel(level string) {
switch level {
case "error":
log.SetLevel(log.ErrorLevel)
case "warn":
log.SetLevel(log.WarnLevel)
case "info":
log.SetLevel(log.InfoLevel)
case "debug":
log.SetLevel(log.DebugLevel)
default:
log.Warnln("Unrecognized minimum log level; using 'info' as default")
}
}
var clock Clock = &SystemClock{}
func init() {
config.SetConfigName("ha_cluster_exporter")
config.AddConfigPath("./")
config.AddConfigPath("$HOME/.config/")
config.AddConfigPath("/etc/")
config.AddConfigPath("/usr/etc/")
flag.String("port", "9002", "The port number to listen on for HTTP requests")
flag.String("address", "0.0.0.0", "The address to listen on for HTTP requests")
flag.String("log-level", "info", "The minimum logging level; levels are, in ascending order: debug, info, warn, error")
flag.String("crm-mon-path", "/usr/sbin/crm_mon", "path to crm_mon executable")
flag.String("cibadmin-path", "/usr/sbin/cibadmin", "path to cibadmin executable")
flag.String("corosync-cfgtoolpath-path", "/usr/sbin/corosync-cfgtool", "path to corosync-cfgtool executable")
flag.String("corosync-quorumtool-path", "/usr/sbin/corosync-quorumtool", "path to corosync-quorumtool executable")
flag.String("sbd-path", "/usr/sbin/sbd", "path to sbd executable")
flag.String("sbd-config-path", "/etc/sysconfig/sbd", "path to sbd configuration")
flag.String("drbdsetup-path", "/sbin/drbdsetup", "path to drbdsetup executable")
flag.String("drbdsplitbrain-path", "/var/run/drbd/splitbrain", "path to drbd splitbrain hooks temporary files")
err := config.BindPFlags(flag.CommandLine)
if err != nil {
log.Errorf("Could not bind config to CLI flags: %v", err)
}
}
func main() {
var err error
flag.Parse()
err = config.ReadInConfig()
if err != nil {
log.Warn(err)
log.Info("Default config values will be used")
} else {
log.Info("Using config file: ", config.ConfigFileUsed())
}
loglevel(config.GetString("log-level"))
pacemakerCollector, err := NewPacemakerCollector(
config.GetString("crm-mon-path"),
config.GetString("cibadmin-path"),
)
if err != nil {
log.Warn(err)
} else {
prometheus.MustRegister(pacemakerCollector)
log.Info("Pacemaker collector registered")
}
corosyncCollector, err := NewCorosyncCollector(
config.GetString("corosync-cfgtoolpath-path"),
config.GetString("corosync-quorumtool-path"),
)
if err != nil {
log.Warn(err)
} else {
prometheus.MustRegister(corosyncCollector)
log.Info("Corosync collector registered")
}
sbdCollector, err := NewSbdCollector(
config.GetString("sbd-path"),
config.GetString("sbd-config-path"),
)
if err != nil {
log.Warn(err)
} else {
prometheus.MustRegister(sbdCollector)
log.Info("SBD collector registered")
}
drbdCollector, err := NewDrbdCollector(config.GetString("drbdsetup-path"), config.GetString("drbdsplitbrain-path"))
if err != nil {
log.Warn(err)
} else {
prometheus.MustRegister(drbdCollector)
log.Info("DRBD collector registered")
}
fullListenAddress := fmt.Sprintf("%s:%s", config.Get("address"), config.Get("port"))
http.HandleFunc("/", landingpage)
http.Handle("/metrics", promhttp.Handler())
log.Infof("Serving metrics on %s", fullListenAddress)
log.Fatal(http.ListenAndServe(fullListenAddress, nil))
}
ha_cluster_exporter-0.4.0/ha_cluster_exporter.service 0000664 0000000 0000000 00000000436 13574721033 0023247 0 ustar 00root root 0000000 0000000 [Unit]
Description=Prometheus exporter for Pacemaker HA clusters metrics
Wants=pacemaker.service
After=network.target pacemaker.service
[Service]
Type=simple
Restart=always
ExecStart=/usr/bin/ha_cluster_exporter
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target
ha_cluster_exporter-0.4.0/ha_cluster_exporter.yaml 0000664 0000000 0000000 00000000533 13574721033 0022547 0 ustar 00root root 0000000 0000000 # sample config
port: "9002"
addres: "0.0.0.0"
log-level: "info"
crm-mon-path: "/usr/sbin/crm_mon"
cibadmin-path: "/usr/sbin/cibadmin"
corosync-cfgtoolpath-path: "/usr/sbin/corosync-cfgtool"
corosync-quorumtool-path: "/usr/sbin/corosync-quorumtool"
sbd-path: "/usr/sbin/sbd"
sbd-config-path: "/etc/sysconfig/sbd"
drbdsetup-path: "/sbin/drbdsetup"
ha_cluster_exporter-0.4.0/pacemaker_metrics.go 0000664 0000000 0000000 00000024501 13574721033 0021610 0 ustar 00root root 0000000 0000000 package main
import (
"encoding/xml"
"math"
"os/exec"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
// *** crm_mon XML unserialization structures
type pacemakerStatus struct {
Version string `xml:"version,attr"`
Summary summary `xml:"summary"`
Nodes struct {
Node []node `xml:"node"`
} `xml:"nodes"`
NodeHistory struct {
Node []struct {
Name string `xml:"name,attr"`
ResourceHistory []struct {
Name string `xml:"id,attr"`
MigrationThreshold int `xml:"migration-threshold,attr"`
FailCount int `xml:"fail-count,attr"`
} `xml:"resource_history"`
} `xml:"node"`
} `xml:"node_history"`
}
type summary struct {
Nodes struct {
Number int `xml:"number,attr"`
} `xml:"nodes_configured"`
LastChange struct {
Time string `xml:"time,attr"`
} `xml:"last_change"`
Resources struct {
Number int `xml:"number,attr"`
Disabled int `xml:"disabled,attr"`
Blocked int `xml:"blocked,attr"`
} `xml:"resources_configured"`
ClusterOptions struct {
StonithEnabled bool `xml:"stonith-enabled,attr"`
} `xml:"cluster_options"`
}
type node struct {
Name string `xml:"name,attr"`
ID string `xml:"id,attr"`
Online bool `xml:"online,attr"`
Standby bool `xml:"standby,attr"`
StandbyOnFail bool `xml:"standby_onfail,attr"`
Maintenance bool `xml:"maintenance,attr"`
Pending bool `xml:"pending,attr"`
Unclean bool `xml:"unclean,attr"`
Shutdown bool `xml:"shutdown,attr"`
ExpectedUp bool `xml:"expected_up,attr"`
DC bool `xml:"is_dc,attr"`
ResourcesRunning int `xml:"resources_running,attr"`
Type string `xml:"type,attr"`
Resources []resource `xml:"resource"`
}
type resource struct {
ID string `xml:"id,attr"`
Agent string `xml:"resource_agent,attr"`
Role string `xml:"role,attr"`
Active bool `xml:"active,attr"`
Orphaned bool `xml:"orphaned,attr"`
Blocked bool `xml:"blocked,attr"`
Managed bool `xml:"managed,attr"`
Failed bool `xml:"failed,attr"`
FailureIgnored bool `xml:"failure_ignored,attr"`
NodesRunningOn int `xml:"nodes_running_on,attr"`
}
// Pacemaker CIB is queried from cibadmin and unmarshaled from XML
type CIB struct {
Configuration struct {
Constraints struct {
RscLocations []struct {
Id string `xml:"id,attr"`
Node string `xml:"node,attr"`
Resource string `xml:"rsc,attr"`
Role string `xml:"role,attr"`
Score string `xml:"score,attr"`
} `xml:"rsc_location"`
} `xml:"constraints"`
} `xml:"configuration"`
}
var (
pacemakerMetrics = metricDescriptors{
// the map key will function as an identifier of the metric throughout the rest of the code;
// it is arbitrary, but by convention we use the actual metric name
"nodes": NewMetricDesc("pacemaker", "nodes", "The nodes in the cluster; one line per name, per status", []string{"node", "type", "status"}),
"nodes_total": NewMetricDesc("pacemaker", "nodes_total", "Total number of nodes in the cluster", nil),
"resources": NewMetricDesc("pacemaker", "resources", "The resources in the cluster; one line per id, per status", []string{"node", "resource", "role", "managed", "status"}),
"resources_total": NewMetricDesc("pacemaker", "resources_total", "Total number of resources in the cluster", nil),
"stonith_enabled": NewMetricDesc("pacemaker", "stonith_enabled", "Whether or not stonith is enabled", nil),
"fail_count": NewMetricDesc("pacemaker", "fail_count", "The Fail count number per node and resource id", []string{"node", "resource"}),
"migration_threshold": NewMetricDesc("pacemaker", "migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"}),
"config_last_change": NewMetricDesc("pacemaker", "config_last_change", "The timestamp of the last change of the cluster configuration", nil),
"location_constraints": NewMetricDesc("pacemaker", "location_constraints", "Resource location constraints. The value indicates the score.", []string{"constraint", "node", "resource", "role"}),
}
)
func NewPacemakerCollector(crmMonPath string, cibAdminPath string) (*pacemakerCollector, error) {
err := CheckExecutables(crmMonPath, cibAdminPath)
if err != nil {
return nil, errors.Wrap(err, "could not initialize Pacemaker collector")
}
return &pacemakerCollector{
DefaultCollector{
metrics: pacemakerMetrics,
},
crmMonPath,
cibAdminPath,
}, nil
}
type pacemakerCollector struct {
DefaultCollector
crmMonPath string
cibAdminPath string
}
func (c *pacemakerCollector) Collect(ch chan<- prometheus.Metric) {
c.mutex.Lock()
defer c.mutex.Unlock()
log.Infoln("Collecting pacemaker metrics...")
pacemakerStatus, err := c.getPacemakerStatus()
if err != nil {
log.Warnln(err)
return
}
cib, err := c.getCIB()
if err != nil {
log.Warnln(err)
return
}
var stonithEnabled float64
if pacemakerStatus.Summary.ClusterOptions.StonithEnabled {
stonithEnabled = 1
}
ch <- c.makeGaugeMetric("nodes_total", float64(pacemakerStatus.Summary.Nodes.Number))
ch <- c.makeGaugeMetric("resources_total", float64(pacemakerStatus.Summary.Resources.Number))
ch <- c.makeGaugeMetric("stonith_enabled", stonithEnabled)
c.recordNodeMetrics(pacemakerStatus, ch)
c.recordFailCountMetrics(pacemakerStatus, ch)
c.recordMigrationThresholdMetrics(pacemakerStatus, ch)
c.recordResourceAgentsChanges(pacemakerStatus, ch)
c.recordConstraintsMetrics(cib, ch)
}
func (c *pacemakerCollector) getPacemakerStatus() (pacemakerStatus, error) {
var pacemakerStatus pacemakerStatus
pacemakerStatusXML, err := exec.Command(c.crmMonPath, "-X", "--group-by-node", "--inactive").Output()
if err != nil {
return pacemakerStatus, errors.Wrap(err, "error while executing crm_mon")
}
pacemakerStatus, err = parsePacemakerStatus(pacemakerStatusXML)
if err != nil {
return pacemakerStatus, errors.Wrap(err, "error while parsing crm_mon XML output")
}
return pacemakerStatus, nil
}
func parsePacemakerStatus(pacemakerXMLRaw []byte) (pacemakerStatus, error) {
var pacemakerStatus pacemakerStatus
err := xml.Unmarshal(pacemakerXMLRaw, &pacemakerStatus)
if err != nil {
return pacemakerStatus, errors.Wrap(err, "could not parse Pacemaker status from XML")
}
return pacemakerStatus, nil
}
func (c *pacemakerCollector) recordNodeMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
for _, node := range pacemakerStatus.Nodes.Node {
nodeStatuses := map[string]bool{
"online": node.Online,
"standby": node.Standby,
"standby_onfail": node.StandbyOnFail,
"maintenance": node.Maintenance,
"pending": node.Pending,
"unclean": node.Unclean,
"shutdown": node.Shutdown,
"expected_up": node.ExpectedUp,
"dc": node.DC,
}
var nodeType string
switch node.Type {
case "member", "ping", "remote":
nodeType = node.Type
break
default:
nodeType = "unknown"
}
for nodeStatus, isActive := range nodeStatuses {
if isActive {
ch <- c.makeGaugeMetric("nodes", float64(1), node.Name, nodeType, nodeStatus)
}
}
c.recordResourcesMetrics(node, ch)
}
}
func (c *pacemakerCollector) recordResourcesMetrics(node node, ch chan<- prometheus.Metric) {
for _, resource := range node.Resources {
resourceStatuses := map[string]bool{
"active": resource.Active,
"orphaned": resource.Orphaned,
"blocked": resource.Blocked,
"failed": resource.Failed,
"failure_ignored": resource.FailureIgnored,
}
for resourceStatus, isActive := range resourceStatuses {
if isActive {
ch <- c.makeGaugeMetric(
"resources",
float64(1),
node.Name,
resource.ID,
strings.ToLower(resource.Role),
strconv.FormatBool(resource.Managed),
resourceStatus)
}
}
}
}
func (c *pacemakerCollector) recordFailCountMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
for _, node := range pacemakerStatus.NodeHistory.Node {
for _, resHistory := range node.ResourceHistory {
failCount := float64(resHistory.FailCount)
// if value is 1000000 this is a special value in pacemaker which is infinity fail count
if resHistory.FailCount >= 1000000 {
failCount = math.Inf(1)
}
ch <- c.makeGaugeMetric("fail_count", failCount, node.Name, resHistory.Name)
}
}
}
func (c *pacemakerCollector) recordResourceAgentsChanges(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
t, err := time.Parse(time.ANSIC, pacemakerStatus.Summary.LastChange.Time)
if err != nil {
log.Warnln(err)
return
}
// we record the timestamp of the last change as a float counter metric, which is in turn timestamped with the time it was checked
ch <- prometheus.NewMetricWithTimestamp(clock.Now(), prometheus.MustNewConstMetric(c.metrics["config_last_change"], prometheus.CounterValue, float64(t.Unix())))
}
func (c *pacemakerCollector) recordMigrationThresholdMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
for _, node := range pacemakerStatus.NodeHistory.Node {
for _, resHistory := range node.ResourceHistory {
ch <- c.makeGaugeMetric("migration_threshold", float64(resHistory.MigrationThreshold), node.Name, resHistory.Name)
}
}
}
func (c *pacemakerCollector) getCIB() (CIB, error) {
var cib CIB
cibXML, err := exec.Command(c.cibAdminPath, "--query", "--local").Output()
if err != nil {
return cib, errors.Wrap(err, "error while executing cibadmin")
}
err = xml.Unmarshal(cibXML, &cib)
if err != nil {
return cib, errors.Wrap(err, "could not parse cibadmin status from XML")
}
return cib, nil
}
func (c *pacemakerCollector) recordConstraintsMetrics(cib CIB, ch chan<- prometheus.Metric) {
for _, constraint := range cib.Configuration.Constraints.RscLocations {
var constraintScore float64
switch constraint.Score {
case "INFINITY":
constraintScore = math.Inf(1)
case "-INFINITY":
constraintScore = math.Inf(-1)
default:
s, _ := strconv.Atoi(constraint.Score)
constraintScore = float64(s)
}
ch <- c.makeGaugeMetric("location_constraints", constraintScore, constraint.Id, constraint.Node, constraint.Resource, strings.ToLower(constraint.Role))
}
}
ha_cluster_exporter-0.4.0/sbd_metrics.go 0000664 0000000 0000000 00000007341 13574721033 0020433 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strings"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
const SBD_STATUS_UNHEALTHY = 0
const SBD_STATUS_HEALTHY = 1
var (
sbdMetrics = metricDescriptors{
// the map key will function as an identifier of the metric throughout the rest of the code;
// it is arbitrary, but by convention we use the actual metric name
"device_status": NewMetricDesc("sbd", "device_status", "Whether or not an SBD device is healthy; one line per device", []string{"device"}),
"devices_total": NewMetricDesc("sbd", "devices_total", "Total count of configured SBD devices", nil),
}
)
func NewSbdCollector(sbdPath string, sbdConfigPath string) (*sbdCollector, error) {
err := CheckExecutables(sbdPath)
if err != nil {
return nil, errors.Wrap(err, "could not initialize SBD collector")
}
if _, err := os.Stat(sbdConfigPath); os.IsNotExist(err) {
return nil, errors.Errorf("could not initialize SBD collector: '%s' does not exist", sbdConfigPath)
}
return &sbdCollector{
DefaultCollector{
metrics: sbdMetrics,
},
sbdPath,
sbdConfigPath,
}, nil
}
type sbdCollector struct {
DefaultCollector
sbdPath string
sbdConfigPath string
}
func (c *sbdCollector) Collect(ch chan<- prometheus.Metric) {
c.mutex.Lock()
defer c.mutex.Unlock()
log.Infoln("Collecting SBD metrics...")
sbdConfiguration, err := readSdbFile(c.sbdConfigPath)
if err != nil {
log.Warnln(err)
return
}
sbdDevices := getSbdDevices(sbdConfiguration)
ch <- c.makeGaugeMetric("devices_total", float64(len(sbdDevices)))
sbdStatuses := c.getSbdDeviceStatuses(sbdDevices)
for sbdDev, sbdStatus := range sbdStatuses {
ch <- c.makeGaugeMetric("device_status", sbdStatus, sbdDev)
}
}
func readSdbFile(sbdConfigPath string) ([]byte, error) {
sbdConfFile, err := os.Open(sbdConfigPath)
if err != nil {
return nil, fmt.Errorf("could not open sbd config file %s", err)
}
defer sbdConfFile.Close()
sbdConfigRaw, err := ioutil.ReadAll(sbdConfFile)
if err != nil {
return nil, fmt.Errorf("could not read sbd config file %s", err)
}
return sbdConfigRaw, nil
}
// retrieve a list of sbd devices from the config file contents
func getSbdDevices(sbdConfigRaw []byte) []string {
// The following regex matches lines like SBD_DEVICE="/dev/foo" or SBD_DEVICE=/dev/foo;/dev/bar
// It captures all the colon separated device names, without double quotes, into a capture group
// It allows for free indentation, trailing spaces and end of lines, and it will ignore commented lines
// Unbalanced double quotes are not checked and they will still produce a match
// If multiple matching lines are present, only the first will be used
// The single device name pattern is `[\w-/]+`, which is pretty relaxed
regex := regexp.MustCompile(`(?m)^\s*SBD_DEVICE="?((?:[\w-/]+;?)+)"?\s*$`)
sbdDevicesLine := regex.FindStringSubmatch(string(sbdConfigRaw))
// if SBD_DEVICE line could not be found, return 0 devices
if sbdDevicesLine == nil {
return nil
}
// split the first capture group, e.g. `/dev/foo;/dev/bar`; the 0th element is always the whole line
sbdDevices := strings.Split(sbdDevicesLine[1], ";")
return sbdDevices
}
// this function takes a list of sbd devices and returns
// a map of SBD device names with 1 if healthy, 0 if not
func (c *sbdCollector) getSbdDeviceStatuses(sbdDevices []string) map[string]float64 {
sbdStatuses := make(map[string]float64)
for _, sbdDev := range sbdDevices {
_, err := exec.Command(c.sbdPath, "-d", sbdDev, "dump").Output()
// in case of error the device is not healthy
if err != nil {
sbdStatuses[sbdDev] = SBD_STATUS_UNHEALTHY
} else {
sbdStatuses[sbdDev] = SBD_STATUS_HEALTHY
}
}
return sbdStatuses
}