carbon-0.9.12/0000755000076400007640000000000012205241574012747 5ustar vagrantvagrantcarbon-0.9.12/bin/0000755000076400007640000000000012205170400013504 5ustar vagrantvagrantcarbon-0.9.12/bin/carbon-aggregator.py0000755000076400007640000000210112204707552017454 0ustar vagrantvagrant#!/usr/bin/env python """Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import sys import os.path # Figure out where we're installed BIN_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(BIN_DIR) # Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from # source. LIB_DIR = os.path.join(ROOT_DIR, "lib") sys.path.insert(0, LIB_DIR) from carbon.util import run_twistd_plugin from carbon.exceptions import CarbonConfigException try: run_twistd_plugin(__file__) except CarbonConfigException, exc: raise SystemExit(str(exc)) carbon-0.9.12/bin/carbon-cache.py0000755000076400007640000000210112204707552016375 0ustar vagrantvagrant#!/usr/bin/env python """Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import sys import os.path # Figure out where we're installed BIN_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(BIN_DIR) # Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from # source. LIB_DIR = os.path.join(ROOT_DIR, "lib") sys.path.insert(0, LIB_DIR) from carbon.util import run_twistd_plugin from carbon.exceptions import CarbonConfigException try: run_twistd_plugin(__file__) except CarbonConfigException, exc: raise SystemExit(str(exc)) carbon-0.9.12/bin/carbon-client.py0000755000076400007640000001034512204707552016621 0ustar vagrantvagrant#!/usr/bin/env python """Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import sys import imp from os.path import dirname, join, abspath, exists from optparse import OptionParser # Figure out where we're installed BIN_DIR = dirname(abspath(__file__)) ROOT_DIR = dirname(BIN_DIR) CONF_DIR = join(ROOT_DIR, 'conf') default_relayrules = join(CONF_DIR, 'relay-rules.conf') # Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from # source. LIB_DIR = join(ROOT_DIR, 'lib') sys.path.insert(0, LIB_DIR) try: from twisted.internet import epollreactor epollreactor.install() except ImportError: pass from twisted.internet import stdio, reactor, defer from twisted.protocols.basic import LineReceiver from carbon.routers import ConsistentHashingRouter, RelayRulesRouter from carbon.client import CarbonClientManager from carbon import log, events option_parser = OptionParser(usage="%prog [options] ...") option_parser.add_option('--debug', action='store_true', help="Log debug info to stdout") option_parser.add_option('--keyfunc', help="Use a custom key function (path/to/module.py:myFunc)") option_parser.add_option('--replication', type='int', default=1, help='Replication factor') option_parser.add_option('--routing', default='consistent-hashing', help='Routing method: "consistent-hashing" (default) or "relay"') option_parser.add_option('--relayrules', default=default_relayrules, help='relay-rules.conf file to use for relay routing') options, args = option_parser.parse_args() if not args: print 'At least one host:port destination required\n' option_parser.print_usage() raise SystemExit(1) if options.routing not in ('consistent-hashing', 'relay'): print "Invalid --routing value, must be one of:" print " consistent-hashing" print " relay" raise SystemExit(1) destinations = [] for arg in args: parts = arg.split(':', 2) host = parts[0] port = int(parts[1]) if len(parts) > 2: instance = parts[2] else: instance = None destinations.append( (host, port, instance) ) if options.debug: log.logToStdout() log.setDebugEnabled(True) defer.setDebugging(True) if options.routing == 'consistent-hashing': router = ConsistentHashingRouter(options.replication) elif options.routing == 'relay': if exists(options.relayrules): router = RelayRulesRouter(options.relayrules) else: print "relay rules file %s does not exist" % options.relayrules raise SystemExit(1) client_manager = CarbonClientManager(router) reactor.callWhenRunning(client_manager.startService) if options.keyfunc: router.setKeyFunctionFromModule(options.keyfunc) firstConnectAttempts = [client_manager.startClient(dest) for dest in destinations] firstConnectsAttempted = defer.DeferredList(firstConnectAttempts) class StdinMetricsReader(LineReceiver): delimiter = '\n' def lineReceived(self, line): #log.msg("[DEBUG] lineReceived(): %s" % line) try: (metric, value, timestamp) = line.split() datapoint = (float(timestamp), float(value)) assert datapoint[1] == datapoint[1] # filter out NaNs client_manager.sendDatapoint(metric, datapoint) except: log.err(None, 'Dropping invalid line: %s' % line) def connectionLost(self, reason): log.msg('stdin disconnected') def startShutdown(results): log.msg("startShutdown(%s)" % str(results)) allStopped = client_manager.stopAllClients() allStopped.addCallback(shutdown) firstConnectsAttempted.addCallback(startShutdown) stdio.StandardIO( StdinMetricsReader() ) exitCode = 0 def shutdown(results): global exitCode for success, result in results: if not success: exitCode = 1 break if reactor.running: reactor.stop() reactor.run() raise SystemExit(exitCode) carbon-0.9.12/bin/carbon-relay.py0000755000076400007640000000210112204707552016446 0ustar vagrantvagrant#!/usr/bin/env python """Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import sys import os.path # Figure out where we're installed BIN_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(BIN_DIR) # Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from # source. LIB_DIR = os.path.join(ROOT_DIR, "lib") sys.path.insert(0, LIB_DIR) from carbon.util import run_twistd_plugin from carbon.exceptions import CarbonConfigException try: run_twistd_plugin(__file__) except CarbonConfigException, exc: raise SystemExit(str(exc)) carbon-0.9.12/bin/validate-storage-schemas.py0000755000076400007640000000430012204707552020747 0ustar vagrantvagrant#!/usr/bin/env python """Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import sys import whisper from os.path import dirname, exists, join, realpath from ConfigParser import ConfigParser if len(sys.argv) == 2: SCHEMAS_FILE = sys.argv[1] print "Loading storage-schemas configuration from: '%s'" % SCHEMAS_FILE else: SCHEMAS_FILE = realpath(join(dirname(__file__), '..', 'conf', 'storage-schemas.conf')) print "Loading storage-schemas configuration from default location at: '%s'" % SCHEMAS_FILE config_parser = ConfigParser() if not config_parser.read(SCHEMAS_FILE): raise SystemExit("Error: Couldn't read config file: %s" % SCHEMAS_FILE) errors_found = 0 for section in config_parser.sections(): print "Section '%s':" % section options = dict(config_parser.items(section)) retentions = options['retentions'].split(',') archives = [] section_failed = False for retention in retentions: try: archives.append(whisper.parseRetentionDef(retention)) except ValueError, e: print " - Error: Section '%s' contains an invalid item in its retention definition ('%s')" % \ (section, retention) print " %s" % e.message section_failed = True if not section_failed: try: whisper.validateArchiveList(archives) except whisper.InvalidConfiguration, e: print " - Error: Section '%s' contains an invalid retention definition ('%s')" % \ (section, ','.join(retentions)) print " %s" % e.message if section_failed: errors_found += 1 else: print " OK" if errors_found: raise SystemExit( "Storage-schemas configuration '%s' failed validation" % SCHEMAS_FILE) print "Storage-schemas configuration '%s' is valid" % SCHEMAS_FILE carbon-0.9.12/conf/0000755000076400007640000000000012205170400013661 5ustar vagrantvagrantcarbon-0.9.12/conf/aggregation-rules.conf.example0000644000076400007640000000270512204707552021622 0ustar vagrantvagrant# The form of each line in this file should be as follows: # # output_template (frequency) = method input_pattern # # This will capture any received metrics that match 'input_pattern' # for calculating an aggregate metric. The calculation will occur # every 'frequency' seconds and the 'method' can specify 'sum' or # 'avg'. The name of the aggregate metric will be derived from # 'output_template' filling in any captured fields from 'input_pattern'. # # For example, if you're metric naming scheme is: # # .applications... # # You could configure some aggregations like so: # # .applications..all.requests (60) = sum .applications..*.requests # .applications..all.latency (60) = avg .applications..*.latency # # As an example, if the following metrics are received: # # prod.applications.apache.www01.requests # prod.applications.apache.www01.requests # # They would all go into the same aggregation buffer and after 60 seconds the # aggregate metric 'prod.applications.apache.all.requests' would be calculated # by summing their values. # # Template components such as will match everything up to the next dot. # To match metric multiple components including the dots, use <> in the # input template: # # .applications..all. (60) = sum .applications..*.<> # # Note that any time this file is modified, it will be re-read automatically. carbon-0.9.12/conf/blacklist.conf.example0000644000076400007640000000042212204707552020145 0ustar vagrantvagrant# This file takes a single regular expression per line # If USE_WHITELIST is set to True in carbon.conf, any metrics received which # match one of these expressions will be dropped # This file is reloaded automatically when changes are made ^some\.noisy\.metric\.prefix\..* carbon-0.9.12/conf/carbon.amqp.conf.example0000644000076400007640000000504212204707552020401 0ustar vagrantvagrant# This is a configuration file with AMQP enabled [cache] LOCAL_DATA_DIR = # Specify the user to drop privileges to # If this is blank carbon runs as the user that invokes it # This user must have write access to the local data directory USER = # Limit the size of the cache to avoid swapping or becoming CPU bound. # Sorts and serving cache queries gets more expensive as the cache grows. # Use the value "inf" (infinity) for an unlimited cache size. MAX_CACHE_SIZE = inf # Limits the number of whisper update_many() calls per second, which effectively # means the number of write requests sent to the disk. This is intended to # prevent over-utilizing the disk and thus starving the rest of the system. # When the rate of required updates exceeds this, then carbon's caching will # take effect and increase the overall throughput accordingly. MAX_UPDATES_PER_SECOND = 1000 # Softly limits the number of whisper files that get created each minute. # Setting this value low (like at 50) is a good way to ensure your graphite # system will not be adversely impacted when a bunch of new metrics are # sent to it. The trade off is that it will take much longer for those metrics' # database files to all get created and thus longer until the data becomes usable. # Setting this value high (like "inf" for infinity) will cause graphite to create # the files quickly but at the risk of slowing I/O down considerably for a while. MAX_CREATES_PER_MINUTE = inf LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2003 UDP_RECEIVER_INTERFACE = 0.0.0.0 UDP_RECEIVER_PORT = 2003 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 CACHE_QUERY_INTERFACE = 0.0.0.0 CACHE_QUERY_PORT = 7002 # Enable AMQP if you want to receve metrics using you amqp broker ENABLE_AMQP = True # Verbose means a line will be logged for every metric received # useful for testing AMQP_VERBOSE = True # your credentials for the amqp server # AMQP_USER = guest # AMQP_PASSWORD = guest # the network settings for the amqp server # AMQP_HOST = localhost # AMQP_PORT = 5672 # if you want to include the metric name as part of the message body # instead of as the routing key, set this to True # AMQP_METRIC_NAME_IN_BODY = False # NOTE: you cannot run both a cache and a relay on the same server # with the default configuration, you have to specify a distinict # interfaces and ports for the listeners. [relay] LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2003 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 CACHE_SERVERS = server1, server2, server3 MAX_QUEUE_SIZE = 10000 carbon-0.9.12/conf/carbon.conf.example0000644000076400007640000003617312205151615017447 0ustar vagrantvagrant[cache] # Configure carbon directories. # # OS environment variables can be used to tell carbon where graphite is # installed, where to read configuration from and where to write data. # # GRAPHITE_ROOT - Root directory of the graphite installation. # Defaults to ../ # GRAPHITE_CONF_DIR - Configuration directory (where this file lives). # Defaults to $GRAPHITE_ROOT/conf/ # GRAPHITE_STORAGE_DIR - Storage directory for whipser/rrd/log/pid files. # Defaults to $GRAPHITE_ROOT/storage/ # # To change other directory paths, add settings to this file. The following # configuration variables are available with these default values: # # STORAGE_DIR = $GRAPHITE_STORAGE_DIR # LOCAL_DATA_DIR = STORAGE_DIR/whisper/ # WHITELISTS_DIR = STORAGE_DIR/lists/ # CONF_DIR = STORAGE_DIR/conf/ # LOG_DIR = STORAGE_DIR/log/ # PID_DIR = STORAGE_DIR/ # # For FHS style directory structures, use: # # STORAGE_DIR = /var/lib/carbon/ # CONF_DIR = /etc/carbon/ # LOG_DIR = /var/log/carbon/ # PID_DIR = /var/run/ # #LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ # Enable daily log rotation. If disabled, a kill -HUP can be used after a manual rotate ENABLE_LOGROTATION = True # Specify the user to drop privileges to # If this is blank carbon runs as the user that invokes it # This user must have write access to the local data directory USER = # # NOTE: The above settings must be set under [relay] and [aggregator] # to take effect for those daemons as well # Limit the size of the cache to avoid swapping or becoming CPU bound. # Sorts and serving cache queries gets more expensive as the cache grows. # Use the value "inf" (infinity) for an unlimited cache size. MAX_CACHE_SIZE = inf # Limits the number of whisper update_many() calls per second, which effectively # means the number of write requests sent to the disk. This is intended to # prevent over-utilizing the disk and thus starving the rest of the system. # When the rate of required updates exceeds this, then carbon's caching will # take effect and increase the overall throughput accordingly. MAX_UPDATES_PER_SECOND = 500 # If defined, this changes the MAX_UPDATES_PER_SECOND in Carbon when a # stop/shutdown is initiated. This helps when MAX_UPDATES_PER_SECOND is # relatively low and carbon has cached a lot of updates; it enables the carbon # daemon to shutdown more quickly. # MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000 # Softly limits the number of whisper files that get created each minute. # Setting this value low (like at 50) is a good way to ensure your graphite # system will not be adversely impacted when a bunch of new metrics are # sent to it. The trade off is that it will take much longer for those metrics' # database files to all get created and thus longer until the data becomes usable. # Setting this value high (like "inf" for infinity) will cause graphite to create # the files quickly but at the risk of slowing I/O down considerably for a while. MAX_CREATES_PER_MINUTE = 50 LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2003 # Set this to True to enable the UDP listener. By default this is off # because it is very common to run multiple carbon daemons and managing # another (rarely used) port for every carbon instance is not fun. ENABLE_UDP_LISTENER = False UDP_RECEIVER_INTERFACE = 0.0.0.0 UDP_RECEIVER_PORT = 2003 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 # Set to false to disable logging of successful connections LOG_LISTENER_CONNECTIONS = True # Per security concerns outlined in Bug #817247 the pickle receiver # will use a more secure and slightly less efficient unpickler. # Set this to True to revert to the old-fashioned insecure unpickler. USE_INSECURE_UNPICKLER = False CACHE_QUERY_INTERFACE = 0.0.0.0 CACHE_QUERY_PORT = 7002 # Set this to False to drop datapoints received after the cache # reaches MAX_CACHE_SIZE. If this is True (the default) then sockets # over which metrics are received will temporarily stop accepting # data until the cache size falls below 95% MAX_CACHE_SIZE. USE_FLOW_CONTROL = True # By default, carbon-cache will log every whisper update and cache hit. This can be excessive and # degrade performance if logging on the same volume as the whisper data is stored. LOG_UPDATES = False LOG_CACHE_HITS = False LOG_CACHE_QUEUE_SORTS = True # The thread that writes metrics to disk can use on of the following strategies # determining the order in which metrics are removed from cache and flushed to # disk. The default option preserves the same behavior as has been historically # available in version 0.9.10. # # sorted - All metrics in the cache will be counted and an ordered list of # them will be sorted according to the number of datapoints in the cache at the # moment of the list's creation. Metrics will then be flushed from the cache to # disk in that order. # # max - The writer thread will always pop and flush the metric from cache # that has the most datapoints. This will give a strong flush preference to # frequently updated metrics and will also reduce random file-io. Infrequently # updated metrics may only ever be persisted to disk at daemon shutdown if # there are a large number of metrics which receive very frequent updates OR if # disk i/o is very slow. # # naive - Metrics will be flushed from the cache to disk in an unordered # fashion. This strategy may be desirable in situations where the storage for # whisper files is solid state, CPU resources are very limited or deference to # the OS's i/o scheduler is expected to compensate for the random write # pattern. # CACHE_WRITE_STRATEGY = sorted # On some systems it is desirable for whisper to write synchronously. # Set this option to True if you'd like to try this. Basically it will # shift the onus of buffering writes from the kernel into carbon's cache. WHISPER_AUTOFLUSH = False # By default new Whisper files are created pre-allocated with the data region # filled with zeros to prevent fragmentation and speed up contiguous reads and # writes (which are common). Enabling this option will cause Whisper to create # the file sparsely instead. Enabling this option may allow a large increase of # MAX_CREATES_PER_MINUTE but may have longer term performance implications # depending on the underlying storage configuration. # WHISPER_SPARSE_CREATE = False # Only beneficial on linux filesystems that support the fallocate system call. # It maintains the benefits of contiguous reads/writes, but with a potentially # much faster creation speed, by allowing the kernel to handle the block # allocation and zero-ing. Enabling this option may allow a large increase of # MAX_CREATES_PER_MINUTE. If enabled on an OS or filesystem that is unsupported # this option will gracefully fallback to standard POSIX file access methods. WHISPER_FALLOCATE_CREATE = True # Enabling this option will cause Whisper to lock each Whisper file it writes # to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when # multiple carbon-cache daemons are writing to the same files # WHISPER_LOCK_WRITES = False # Set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through # USE_WHITELIST = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation # CARBON_METRIC_PREFIX = carbon # CARBON_METRIC_INTERVAL = 60 # Enable AMQP if you want to receve metrics using an amqp broker # ENABLE_AMQP = False # Verbose means a line will be logged for every metric received # useful for testing # AMQP_VERBOSE = False # AMQP_HOST = localhost # AMQP_PORT = 5672 # AMQP_VHOST = / # AMQP_USER = guest # AMQP_PASSWORD = guest # AMQP_EXCHANGE = graphite # AMQP_METRIC_NAME_IN_BODY = False # The manhole interface allows you to SSH into the carbon daemon # and get a python interpreter. BE CAREFUL WITH THIS! If you do # something like time.sleep() in the interpreter, the whole process # will sleep! This is *extremely* helpful in debugging, assuming # you are familiar with the code. If you are not, please don't # mess with this, you are asking for trouble :) # # ENABLE_MANHOLE = False # MANHOLE_INTERFACE = 127.0.0.1 # MANHOLE_PORT = 7222 # MANHOLE_USER = admin # MANHOLE_PUBLIC_KEY = ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE= # Patterns for all of the metrics this machine will store. Read more at # http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings # # Example: store all sales, linux servers, and utilization metrics # BIND_PATTERNS = sales.#, servers.linux.#, #.utilization # # Example: store everything # BIND_PATTERNS = # # To configure special settings for the carbon-cache instance 'b', uncomment this: #[cache:b] #LINE_RECEIVER_PORT = 2103 #PICKLE_RECEIVER_PORT = 2104 #CACHE_QUERY_PORT = 7102 # and any other settings you want to customize, defaults are inherited # from [carbon] section. # You can then specify the --instance=b option to manage this instance [relay] LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2013 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2014 # Set to false to disable logging of successful connections LOG_LISTENER_CONNECTIONS = True # Carbon-relay has several options for metric routing controlled by RELAY_METHOD # # Use relay-rules.conf to route metrics to destinations based on pattern rules #RELAY_METHOD = rules # # Use consistent-hashing for even distribution of metrics between destinations #RELAY_METHOD = consistent-hashing # # Use consistent-hashing but take into account an aggregation-rules.conf shared # by downstream carbon-aggregator daemons. This will ensure that all metrics # that map to a given aggregation rule are sent to the same carbon-aggregator # instance. # Enable this for carbon-relays that send to a group of carbon-aggregators #RELAY_METHOD = aggregated-consistent-hashing RELAY_METHOD = rules # If you use consistent-hashing you can add redundancy by replicating every # datapoint to more than one machine. REPLICATION_FACTOR = 1 # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you # use multiple carbon-cache instances then it would look like this: # # DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b # # The general form is IP:PORT:INSTANCE where the :INSTANCE part is # optional and refers to the "None" instance if omitted. # # Note that if the destinations are all carbon-caches then this should # exactly match the webapp's CARBONLINK_HOSTS setting in terms of # instances listed (order matters!). # # If using RELAY_METHOD = rules, all destinations used in relay-rules.conf # must be defined in this list DESTINATIONS = 127.0.0.1:2004 # This defines the maximum "message size" between carbon daemons. # You shouldn't need to tune this unless you really know what you're doing. MAX_DATAPOINTS_PER_MESSAGE = 500 MAX_QUEUE_SIZE = 10000 # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting # data until the send queues fall below 80% MAX_QUEUE_SIZE. USE_FLOW_CONTROL = True # Set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through # USE_WHITELIST = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation # CARBON_METRIC_PREFIX = carbon # CARBON_METRIC_INTERVAL = 60 [aggregator] LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2023 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2024 # Set to false to disable logging of successful connections LOG_LISTENER_CONNECTIONS = True # If set true, metric received will be forwarded to DESTINATIONS in addition to # the output of the aggregation rules. If set false the carbon-aggregator will # only ever send the output of aggregation. FORWARD_ALL = True # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you # use multiple carbon-cache instances then it would look like this: # # DESTINATIONS = 127.0.0.1:2004:a, 127.0.0.1:2104:b # # The format is comma-delimited IP:PORT:INSTANCE where the :INSTANCE part is # optional and refers to the "None" instance if omitted. # # Note that if the destinations are all carbon-caches then this should # exactly match the webapp's CARBONLINK_HOSTS setting in terms of # instances listed (order matters!). DESTINATIONS = 127.0.0.1:2004 # If you want to add redundancy to your data by replicating every # datapoint to more than one machine, increase this. REPLICATION_FACTOR = 1 # This is the maximum number of datapoints that can be queued up # for a single destination. Once this limit is hit, we will # stop accepting new data if USE_FLOW_CONTROL is True, otherwise # we will drop any subsequently received datapoints. MAX_QUEUE_SIZE = 10000 # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting # data until the send queues fall below 80% MAX_QUEUE_SIZE. USE_FLOW_CONTROL = True # This defines the maximum "message size" between carbon daemons. # You shouldn't need to tune this unless you really know what you're doing. MAX_DATAPOINTS_PER_MESSAGE = 500 # This defines how many datapoints the aggregator remembers for # each metric. Aggregation only happens for datapoints that fall in # the past MAX_AGGREGATION_INTERVALS * intervalSize seconds. MAX_AGGREGATION_INTERVALS = 5 # By default (WRITE_BACK_FREQUENCY = 0), carbon-aggregator will write back # aggregated data points once every rule.frequency seconds, on a per-rule basis. # Set this (WRITE_BACK_FREQUENCY = N) to write back all aggregated data points # every N seconds, independent of rule frequency. This is useful, for example, # to be able to query partially aggregated metrics from carbon-cache without # having to first wait rule.frequency seconds. # WRITE_BACK_FREQUENCY = 0 # Set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through # USE_WHITELIST = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation # CARBON_METRIC_PREFIX = carbon # CARBON_METRIC_INTERVAL = 60 carbon-0.9.12/conf/relay-rules.conf.example0000644000076400007640000000157512204707552020453 0ustar vagrantvagrant# Relay destination rules for carbon-relay. Entries are scanned in order, # and the first pattern a metric matches will cause processing to cease after sending # unless `continue` is set to true # # [name] # pattern = # destinations = # continue = # default: False # # name: Arbitrary unique name to identify the rule # pattern: Regex pattern to match against the metric name # destinations: Comma-separated list of destinations. # ex: 127.0.0.1, 10.1.2.3:2004, 10.1.2.4:2004:a, myserver.mydomain.com # continue: Continue processing rules if this rule matches (default: False) # You must have exactly one section with 'default = true' # Note that all destinations listed must also exist in carbon.conf # in the DESTINATIONS setting in the [relay] section [default] default = true destinations = 127.0.0.1:2004:a, 127.0.0.1:2104:b carbon-0.9.12/conf/rewrite-rules.conf.example0000644000076400007640000000105612204707552021012 0ustar vagrantvagrant# This file defines regular expression patterns that can be used to # rewrite metric names in a search & replace fashion. It consists of two # sections, [pre] and [post]. The rules in the pre section are applied to # metric names as soon as they are received. The post rules are applied # after aggregation has taken place. # # The general form of each rule is as follows: # # regex-pattern = replacement-text # # For example: # # [post] # _sum$ = # _avg$ = # # These rules would strip off a suffix of _sum or _avg from any metric names # after aggregation. carbon-0.9.12/conf/storage-aggregation.conf.example0000644000076400007640000000147312204707552022135 0ustar vagrantvagrant# Aggregation methods for whisper files. Entries are scanned in order, # and first match wins. This file is scanned for changes every 60 seconds # # [name] # pattern = # xFilesFactor = # aggregationMethod = # # name: Arbitrary unique name for the rule # pattern: Regex pattern to match against the metric name # xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur # aggregationMethod: function to apply to data points for aggregation # [min] pattern = \.min$ xFilesFactor = 0.1 aggregationMethod = min [max] pattern = \.max$ xFilesFactor = 0.1 aggregationMethod = max [sum] pattern = \.count$ xFilesFactor = 0 aggregationMethod = sum [default_average] pattern = .* xFilesFactor = 0.5 aggregationMethod = average carbon-0.9.12/conf/storage-schemas.conf.example0000644000076400007640000000075112205151615021261 0ustar vagrantvagrant# Schema definitions for Whisper files. Entries are scanned in order, # and first match wins. This file is scanned for changes every 60 seconds. # # [name] # pattern = regex # retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ... # Carbon's internal metrics. This entry should match what is specified in # CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings [carbon] pattern = ^carbon\. retentions = 60:90d [default_1min_for_1day] pattern = .* retentions = 60s:1d carbon-0.9.12/conf/whitelist.conf.example0000644000076400007640000000047312204707552020217 0ustar vagrantvagrant# This file takes a single regular expression per line # If USE_WHITELIST is set to True in carbon.conf, only metrics received which # match one of these expressions will be persisted. If this file is empty or # missing, all metrics will pass through. # This file is reloaded automatically when changes are made .* carbon-0.9.12/distro/0000755000076400007640000000000012205241574014253 5ustar vagrantvagrantcarbon-0.9.12/distro/redhat/0000755000076400007640000000000012205241622015514 5ustar vagrantvagrantcarbon-0.9.12/distro/redhat/init.d/0000755000076400007640000000000012205241574016707 5ustar vagrantvagrantcarbon-0.9.12/distro/redhat/init.d/carbon-aggregator0000755000076400007640000000431312205241574022222 0ustar vagrantvagrant#!/bin/bash # chkconfig: - 25 75 # description: carbon-aggregator # processname: carbon-aggregator export PYTHONPATH="$GRAPHITE_DIR/lib:$PYTHONPATH" # Source function library. if [ -e /etc/rc.d/init.d/functions ]; then . /etc/rc.d/init.d/functions; fi; CARBON_DAEMON="aggregator" GRAPHITE_DIR="/opt/graphite" INSTANCES=`grep "^\[${CARBON_DAEMON}" ${GRAPHITE_DIR}/conf/carbon.conf | cut -d \[ -f 2 | cut -d \] -f 1 | cut -d : -f 2` function die { echo $1 exit 1 } start(){ cd $GRAPHITE_DIR; for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; echo "Starting carbon-${CARBON_DAEMON}:${INSTANCE}..." bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} start; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } stop(){ cd $GRAPHITE_DIR for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; echo "Stopping carbon-${CARBON_DAEMON}:${INSTANCE}..." bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} stop if [ `sleep 3; /usr/bin/pgrep -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}" | /usr/bin/wc -l` -gt 0 ]; then echo "Carbon did not stop yet. Sleeping longer, then force killing it..."; sleep 20; /usr/bin/pkill -9 -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}"; fi; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } status(){ cd $GRAPHITE_DIR; for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} status; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } case "$1" in start) start ;; stop) stop ;; status) status ;; restart|reload) stop start ;; *) echo $"Usage: $0 {start|stop|restart|status}" exit 1 esac carbon-0.9.12/distro/redhat/init.d/carbon-cache0000755000076400007640000000427412205241574021151 0ustar vagrantvagrant#!/bin/bash # chkconfig: - 25 75 # description: carbon-cache # processname: carbon-cache export PYTHONPATH="$GRAPHITE_DIR/lib:$PYTHONPATH" # Source function library. if [ -e /etc/rc.d/init.d/functions ]; then . /etc/rc.d/init.d/functions; fi; CARBON_DAEMON="cache" GRAPHITE_DIR="/opt/graphite" INSTANCES=`grep "^\[${CARBON_DAEMON}" ${GRAPHITE_DIR}/conf/carbon.conf | cut -d \[ -f 2 | cut -d \] -f 1 | cut -d : -f 2` function die { echo $1 exit 1 } start(){ cd $GRAPHITE_DIR; for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; echo "Starting carbon-${CARBON_DAEMON}:${INSTANCE}..." bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} start; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } stop(){ cd $GRAPHITE_DIR for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; echo "Stopping carbon-${CARBON_DAEMON}:${INSTANCE}..." bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} stop if [ `sleep 3; /usr/bin/pgrep -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}" | /usr/bin/wc -l` -gt 0 ]; then echo "Carbon did not stop yet. Sleeping longer, then force killing it..."; sleep 20; /usr/bin/pkill -9 -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}"; fi; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } status(){ cd $GRAPHITE_DIR; for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} status; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } case "$1" in start) start ;; stop) stop ;; status) status ;; restart|reload) stop start ;; *) echo $"Usage: $0 {start|stop|restart|status}" exit 1 esac carbon-0.9.12/distro/redhat/init.d/carbon-relay0000755000076400007640000000427412205241574021222 0ustar vagrantvagrant#!/bin/bash # chkconfig: - 25 75 # description: carbon-relay # processname: carbon-relay export PYTHONPATH="$GRAPHITE_DIR/lib:$PYTHONPATH" # Source function library. if [ -e /etc/rc.d/init.d/functions ]; then . /etc/rc.d/init.d/functions; fi; CARBON_DAEMON="relay" GRAPHITE_DIR="/opt/graphite" INSTANCES=`grep "^\[${CARBON_DAEMON}" ${GRAPHITE_DIR}/conf/carbon.conf | cut -d \[ -f 2 | cut -d \] -f 1 | cut -d : -f 2` function die { echo $1 exit 1 } start(){ cd $GRAPHITE_DIR; for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; echo "Starting carbon-${CARBON_DAEMON}:${INSTANCE}..." bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} start; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } stop(){ cd $GRAPHITE_DIR for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; echo "Stopping carbon-${CARBON_DAEMON}:${INSTANCE}..." bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} stop if [ `sleep 3; /usr/bin/pgrep -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}" | /usr/bin/wc -l` -gt 0 ]; then echo "Carbon did not stop yet. Sleeping longer, then force killing it..."; sleep 20; /usr/bin/pkill -9 -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}"; fi; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } status(){ cd $GRAPHITE_DIR; for INSTANCE in ${INSTANCES}; do if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then INSTANCE="a"; fi; bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} status; if [ $? -eq 0 ]; then echo_success else echo_failure fi; echo "" done; } case "$1" in start) start ;; stop) stop ;; status) status ;; restart|reload) stop start ;; *) echo $"Usage: $0 {start|stop|restart|status}" exit 1 esac carbon-0.9.12/lib/0000755000076400007640000000000012205170400013502 5ustar vagrantvagrantcarbon-0.9.12/lib/carbon/0000755000076400007640000000000012205170400014746 5ustar vagrantvagrantcarbon-0.9.12/lib/carbon/__init__.py0000644000076400007640000000000012022301500017037 0ustar vagrantvagrantcarbon-0.9.12/lib/carbon/aggregator/0000755000076400007640000000000012205170400017070 5ustar vagrantvagrantcarbon-0.9.12/lib/carbon/aggregator/__init__.py0000644000076400007640000000000012022301500021161 0ustar vagrantvagrantcarbon-0.9.12/lib/carbon/aggregator/buffers.py0000644000076400007640000000601312204707552021113 0ustar vagrantvagrantimport time from twisted.internet.task import LoopingCall from carbon.conf import settings from carbon import log class BufferManager: def __init__(self): self.buffers = {} def __len__(self): return len(self.buffers) def get_buffer(self, metric_path): if metric_path not in self.buffers: log.aggregator("Allocating new metric buffer for %s" % metric_path) self.buffers[metric_path] = MetricBuffer(metric_path) return self.buffers[metric_path] def clear(self): for buffer in self.buffers.values(): buffer.close() self.buffers.clear() class MetricBuffer: __slots__ = ('metric_path', 'interval_buffers', 'compute_task', 'configured', 'aggregation_frequency', 'aggregation_func') def __init__(self, metric_path): self.metric_path = metric_path self.interval_buffers = {} self.compute_task = None self.configured = False self.aggregation_frequency = None self.aggregation_func = None def input(self, datapoint): (timestamp, value) = datapoint interval = timestamp - (timestamp % self.aggregation_frequency) if interval in self.interval_buffers: buffer = self.interval_buffers[interval] else: buffer = self.interval_buffers[interval] = IntervalBuffer(interval) buffer.input(datapoint) def configure_aggregation(self, frequency, func): self.aggregation_frequency = int(frequency) self.aggregation_func = func self.compute_task = LoopingCall(self.compute_value) self.compute_task.start(settings['WRITE_BACK_FREQUENCY'] or frequency, now=False) self.configured = True def compute_value(self): now = int( time.time() ) current_interval = now - (now % self.aggregation_frequency) age_threshold = current_interval - (settings['MAX_AGGREGATION_INTERVALS'] * self.aggregation_frequency) for buffer in self.interval_buffers.values(): if buffer.active: value = self.aggregation_func(buffer.values) datapoint = (buffer.interval, value) state.events.metricGenerated(self.metric_path, datapoint) state.instrumentation.increment('aggregateDatapointsSent') buffer.mark_inactive() if buffer.interval < age_threshold: del self.interval_buffers[buffer.interval] if not self.interval_buffers: self.close() self.configured = False del BufferManager.buffers[self.metric_path] def close(self): if self.compute_task and self.compute_task.running: self.compute_task.stop() @property def size(self): return sum([len(buf.values) for buf in self.interval_buffers.values()]) class IntervalBuffer: __slots__ = ('interval', 'values', 'active') def __init__(self, interval): self.interval = interval self.values = [] self.active = True def input(self, datapoint): self.values.append( datapoint[1] ) self.active = True def mark_inactive(self): self.active = False # Shared importable singleton BufferManager = BufferManager() # Avoid import circularity from carbon import state carbon-0.9.12/lib/carbon/aggregator/receiver.py0000644000076400007640000000175012205151615021260 0ustar vagrantvagrantfrom carbon.instrumentation import increment from carbon.aggregator.rules import RuleManager from carbon.aggregator.buffers import BufferManager from carbon.conf import settings from carbon.rewrite import RewriteRuleManager from carbon import events def process(metric, datapoint): increment('datapointsReceived') for rule in RewriteRuleManager.preRules: metric = rule.apply(metric) aggregate_metrics = [] for rule in RuleManager.rules: aggregate_metric = rule.get_aggregate_metric(metric) if aggregate_metric is None: continue else: aggregate_metrics.append(aggregate_metric) buffer = BufferManager.get_buffer(aggregate_metric) if not buffer.configured: buffer.configure_aggregation(rule.frequency, rule.aggregation_func) buffer.input(datapoint) for rule in RewriteRuleManager.postRules: metric = rule.apply(metric) if settings['FORWARD_ALL'] and metric not in aggregate_metrics: events.metricGenerated(metric, datapoint) carbon-0.9.12/lib/carbon/aggregator/rules.py0000644000076400007640000000764512204707552020625 0ustar vagrantvagrantimport time import re from os.path import exists, getmtime from twisted.internet.task import LoopingCall from carbon import log from carbon.aggregator.buffers import BufferManager class RuleManager: def __init__(self): self.rules = [] self.rules_file = None self.read_task = LoopingCall(self.read_rules) self.rules_last_read = 0.0 def clear(self): self.rules = [] def read_from(self, rules_file): self.rules_file = rules_file self.read_rules() self.read_task.start(10, now=False) def read_rules(self): if not exists(self.rules_file): self.clear() return # Only read if the rules file has been modified try: mtime = getmtime(self.rules_file) except: log.err("Failed to get mtime of %s" % self.rules_file) return if mtime <= self.rules_last_read: return # Read new rules log.aggregator("reading new aggregation rules from %s" % self.rules_file) new_rules = [] for line in open(self.rules_file): line = line.strip() if line.startswith('#') or not line: continue rule = self.parse_definition(line) new_rules.append(rule) log.aggregator("clearing aggregation buffers") BufferManager.clear() self.rules = new_rules self.rules_last_read = mtime def parse_definition(self, line): try: left_side, right_side = line.split('=', 1) output_pattern, frequency = left_side.split() method, input_pattern = right_side.split() frequency = int( frequency.lstrip('(').rstrip(')') ) return AggregationRule(input_pattern, output_pattern, method, frequency) except: log.err("Failed to parse line: %s" % line) raise class AggregationRule: def __init__(self, input_pattern, output_pattern, method, frequency): self.input_pattern = input_pattern self.output_pattern = output_pattern self.method = method self.frequency = int(frequency) if method not in AGGREGATION_METHODS: raise ValueError("Invalid aggregation method '%s'" % method) self.aggregation_func = AGGREGATION_METHODS[method] self.build_regex() self.build_template() self.cache = {} def get_aggregate_metric(self, metric_path): if metric_path in self.cache: return self.cache[metric_path] match = self.regex.match(metric_path) result = None if match: extracted_fields = match.groupdict() try: result = self.output_template % extracted_fields except: log.err("Failed to interpolate template %s with fields %s" % (self.output_template, extracted_fields)) self.cache[metric_path] = result return result def build_regex(self): input_pattern_parts = self.input_pattern.split('.') regex_pattern_parts = [] for input_part in input_pattern_parts: if '<<' in input_part and '>>' in input_part: i = input_part.find('<<') j = input_part.find('>>') pre = input_part[:i] post = input_part[j+2:] field_name = input_part[i+2:j] regex_part = '%s(?P<%s>.+)%s' % (pre, field_name, post) else: i = input_part.find('<') j = input_part.find('>') if i > -1 and j > i: pre = input_part[:i] post = input_part[j+1:] field_name = input_part[i+1:j] regex_part = '%s(?P<%s>[^.]+)%s' % (pre, field_name, post) elif input_part == '*': regex_part = '[^.]+' else: regex_part = input_part.replace('*', '[^.]*') regex_pattern_parts.append(regex_part) regex_pattern = '\\.'.join(regex_pattern_parts) self.regex = re.compile(regex_pattern) def build_template(self): self.output_template = self.output_pattern.replace('<', '%(').replace('>', ')s') def avg(values): if values: return float( sum(values) ) / len(values) AGGREGATION_METHODS = { 'sum' : sum, 'avg' : avg, 'min' : min, 'max' : max, } # Importable singleton RuleManager = RuleManager() carbon-0.9.12/lib/carbon/amqp0-8.xml0000644000076400007640000007547712022301500016671 0ustar vagrantvagrant carbon-0.9.12/lib/carbon/amqp_listener.py0000644000076400007640000002015712205151615020177 0ustar vagrantvagrant#!/usr/bin/env python """ Copyright 2009 Lucio Torre Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. This is an AMQP client that will connect to the specified broker and read messages, parse them, and post them as metrics. Each message's routing key should be a metric name. The message body should be one or more lines of the form: \n \n ... Where each is a real number and is a UNIX epoch time. This program can be started standalone for testing or using carbon-cache.py (see example config file provided) """ import sys import os import socket from optparse import OptionParser from twisted.python.failure import Failure from twisted.internet.defer import deferredGenerator, waitForDeferred from twisted.internet import reactor from twisted.internet.protocol import ReconnectingClientFactory from txamqp.protocol import AMQClient from txamqp.client import TwistedDelegate import txamqp.spec try: import carbon except: # this is being run directly, carbon is not installed LIB_DIR = os.path.dirname(os.path.dirname(__file__)) sys.path.insert(0, LIB_DIR) import carbon.protocols #satisfy import order requirements from carbon.conf import settings from carbon import log, events, instrumentation HOSTNAME = socket.gethostname().split('.')[0] class AMQPGraphiteProtocol(AMQClient): """This is the protocol instance that will receive and post metrics.""" consumer_tag = "graphite_consumer" @deferredGenerator def connectionMade(self): AMQClient.connectionMade(self) log.listener("New AMQP connection made") self.setup() wfd = waitForDeferred(self.receive_loop()) yield wfd @deferredGenerator def setup(self): exchange = self.factory.exchange_name d = self.authenticate(self.factory.username, self.factory.password) wfd = waitForDeferred(d) yield wfd wfd = waitForDeferred(self.channel(1)) yield wfd chan = wfd.getResult() wfd = waitForDeferred(chan.channel_open()) yield wfd # declare the exchange and queue d = chan.exchange_declare(exchange=exchange, type="topic", durable=True, auto_delete=False) wfd = waitForDeferred(d) yield wfd # we use a private queue to avoid conflicting with existing bindings wfd = waitForDeferred(chan.queue_declare(exclusive=True)) yield wfd reply = wfd.getResult() my_queue = reply.queue # bind each configured metric pattern for bind_pattern in settings.BIND_PATTERNS: log.listener("binding exchange '%s' to queue '%s' with pattern %s" \ % (exchange, my_queue, bind_pattern)) d = chan.queue_bind(exchange=exchange, queue=my_queue, routing_key=bind_pattern) wfd = waitForDeferred(d) yield wfd d = chan.basic_consume(queue=my_queue, no_ack=True, consumer_tag=self.consumer_tag) wfd = waitForDeferred(d) yield wfd @deferredGenerator def receive_loop(self): wfd = waitForDeferred(self.queue(self.consumer_tag)) yield wfd queue = wfd.getResult() while True: wfd = waitForDeferred(queue.get()) yield wfd msg = wfd.getResult() self.processMessage(msg) def processMessage(self, message): """Parse a message and post it as a metric.""" if self.factory.verbose: log.listener("Message received: %s" % (message,)) metric = message.routing_key for line in message.content.body.split("\n"): line = line.strip() if not line: continue try: if settings.get("AMQP_METRIC_NAME_IN_BODY", False): metric, value, timestamp = line.split() else: value, timestamp = line.split() datapoint = ( float(timestamp), float(value) ) if datapoint[1] != datapoint[1]: # filter out NaN values continue except ValueError: log.listener("invalid message line: %s" % (line,)) continue events.metricReceived(metric, datapoint) if self.factory.verbose: log.listener("Metric posted: %s %s %s" % (metric, value, timestamp,)) class AMQPReconnectingFactory(ReconnectingClientFactory): """The reconnecting factory. Knows how to create the extended client and how to keep trying to connect in case of errors.""" protocol = AMQPGraphiteProtocol def __init__(self, username, password, delegate, vhost, spec, channel, exchange_name, verbose): self.username = username self.password = password self.delegate = delegate self.vhost = vhost self.spec = spec self.channel = channel self.exchange_name = exchange_name self.verbose = verbose def buildProtocol(self, addr): p = self.protocol(self.delegate, self.vhost, self.spec) p.factory = self return p def createAMQPListener(username, password, vhost, exchange_name, spec=None, channel=1, verbose=False): """ Create an C{AMQPReconnectingFactory} configured with the specified options. """ # use provided spec if not specified if not spec: spec = txamqp.spec.load(os.path.normpath( os.path.join(os.path.dirname(__file__), 'amqp0-8.xml'))) delegate = TwistedDelegate() factory = AMQPReconnectingFactory(username, password, delegate, vhost, spec, channel, exchange_name, verbose=verbose) return factory def startReceiver(host, port, username, password, vhost, exchange_name, spec=None, channel=1, verbose=False): """ Starts a twisted process that will read messages on the amqp broker and post them as metrics. """ factory = createAMQPListener(username, password, vhost, exchange_name, spec=spec, channel=channel, verbose=verbose) reactor.connectTCP(host, port, factory) def main(): parser = OptionParser() parser.add_option("-t", "--host", dest="host", help="host name", metavar="HOST", default="localhost") parser.add_option("-p", "--port", dest="port", type=int, help="port number", metavar="PORT", default=5672) parser.add_option("-u", "--user", dest="username", help="username", metavar="USERNAME", default="guest") parser.add_option("-w", "--password", dest="password", help="password", metavar="PASSWORD", default="guest") parser.add_option("-V", "--vhost", dest="vhost", help="vhost", metavar="VHOST", default="/") parser.add_option("-e", "--exchange", dest="exchange", help="exchange", metavar="EXCHANGE", default="graphite") parser.add_option("-v", "--verbose", dest="verbose", help="verbose", default=False, action="store_true") (options, args) = parser.parse_args() startReceiver(options.host, options.port, options.username, options.password, vhost=options.vhost, exchange_name=options.exchange, verbose=options.verbose) reactor.run() if __name__ == "__main__": main() carbon-0.9.12/lib/carbon/amqp_publisher.py0000644000076400007640000001060312205151615020342 0ustar vagrantvagrant#!/usr/bin/env python """ Copyright 2009 Lucio Torre Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Will publish metrics over AMQP """ import os import time from optparse import OptionParser from twisted.python.failure import Failure from twisted.internet.defer import deferredGenerator, waitForDeferred from twisted.internet import reactor, task from twisted.internet.protocol import ClientCreator from txamqp.protocol import AMQClient from txamqp.client import TwistedDelegate from txamqp.content import Content import txamqp.spec @deferredGenerator def writeMetric(metric_path, value, timestamp, host, port, username, password, vhost, exchange, spec=None, channel_number=1, ssl=False): if not spec: spec = txamqp.spec.load(os.path.normpath( os.path.join(os.path.dirname(__file__), 'amqp0-8.xml'))) delegate = TwistedDelegate() connector = ClientCreator(reactor, AMQClient, delegate=delegate, vhost=vhost, spec=spec) if ssl: from twisted.internet.ssl import ClientContextFactory wfd = waitForDeferred(connector.connectSSL(host, port, ClientContextFactory())) yield wfd conn = wfd.getResult() else: wfd = waitForDeferred(connector.connectTCP(host, port)) yield wfd conn = wfd.getResult() wfd = waitForDeferred(conn.authenticate(username, password)) yield wfd wfd = waitForDeferred(conn.channel(channel_number)) yield wfd channel = wfd.getResult() wfd = waitForDeferred(channel.channel_open()) yield wfd wfd = waitForDeferred(channel.exchange_declare(exchange=exchange, type="topic", durable=True, auto_delete=False)) yield wfd message = Content( "%f %d" % (value, timestamp) ) message["delivery mode"] = 2 channel.basic_publish(exchange=exchange, content=message, routing_key=metric_path) wfd = waitForDeferred(channel.channel_close()) yield wfd def main(): parser = OptionParser(usage="%prog [options] [timestamp]") parser.add_option("-t", "--host", dest="host", help="host name", metavar="HOST", default="localhost") parser.add_option("-p", "--port", dest="port", type=int, help="port number", metavar="PORT", default=5672) parser.add_option("-u", "--user", dest="username", help="username", metavar="USERNAME", default="guest") parser.add_option("-w", "--password", dest="password", help="password", metavar="PASSWORD", default="guest") parser.add_option("-v", "--vhost", dest="vhost", help="vhost", metavar="VHOST", default="/") parser.add_option("-s", "--ssl", dest="ssl", help="ssl", metavar="SSL", action="store_true", default=False) parser.add_option("-e", "--exchange", dest="exchange", help="exchange", metavar="EXCHANGE", default="graphite") (options, args) = parser.parse_args() try: metric_path = args[0] value = float(args[1]) if len(args) > 2: timestamp = int(args[2]) else: timestamp = time.time() except: parser.print_usage() raise SystemExit(1) d = writeMetric(metric_path, value, timestamp, options.host, options.port, options.username, options.password, vhost=options.vhost, exchange=options.exchange, ssl=options.ssl) d.addErrback(lambda f: f.printTraceback()) d.addBoth(lambda _: reactor.stop()) reactor.run() if __name__ == "__main__": main() carbon-0.9.12/lib/carbon/cache.py0000644000076400007640000000474412205151615016403 0ustar vagrantvagrant"""Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import time from collections import deque from carbon.conf import settings try: from collections import defaultdict except: from util import defaultdict class _MetricCache(defaultdict): def __init__(self, defaultfactory=deque, method="sorted"): self.method = method if self.method == "sorted": self.queue = self.gen_queue() else: self.queue = False super(_MetricCache, self).__init__(defaultfactory) def gen_queue(self): while True: t = time.time() queue = sorted(self.counts, key=lambda x: x[1]) if settings.LOG_CACHE_QUEUE_SORTS: log.debug("Sorted %d cache queues in %.6f seconds" % (len(queue), time.time() - t)) while queue: yield queue.pop()[0] @property def size(self): return reduce(lambda x, y: x + len(y), self.values(), 0) def store(self, metric, datapoint): self[metric].append(datapoint) if self.isFull(): log.msg("MetricCache is full: self.size=%d" % self.size) state.events.cacheFull() def isFull(self): # Short circuit this test if there is no max cache size, then we don't need # to do the someone expensive work of calculating the current size. return settings.MAX_CACHE_SIZE != float('inf') and self.size >= settings.MAX_CACHE_SIZE def pop(self, metric=None): if not self: raise KeyError(metric) elif not metric and self.method == "max": metric = max(self.items(), key=lambda x: len(x[1]))[0] elif not metric and self.method == "naive": return self.popitem() elif not metric and self.method == "sorted": metric = self.queue.next() datapoints = (metric, super(_MetricCache, self).pop(metric)) return datapoints @property def counts(self): return [(metric, len(datapoints)) for (metric, datapoints) in self.items()] # Ghetto singleton MetricCache = _MetricCache(method=settings.CACHE_WRITE_STRATEGY) # Avoid import circularities from carbon import log, state carbon-0.9.12/lib/carbon/client.py0000644000076400007640000002166612205151615016620 0ustar vagrantvagrantfrom twisted.application.service import Service from twisted.internet import reactor from twisted.internet.defer import Deferred, DeferredList from twisted.internet.protocol import ReconnectingClientFactory from twisted.protocols.basic import Int32StringReceiver from carbon.conf import settings from carbon.util import pickle from carbon import log, state, instrumentation SEND_QUEUE_LOW_WATERMARK = settings.MAX_QUEUE_SIZE * 0.8 class CarbonClientProtocol(Int32StringReceiver): def connectionMade(self): log.clients("%s::connectionMade" % self) self.paused = False self.connected = True self.transport.registerProducer(self, streaming=True) # Define internal metric names self.destinationName = self.factory.destinationName self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName self.sent = 'destinations.%s.sent' % self.destinationName self.factory.connectionMade.callback(self) self.factory.connectionMade = Deferred() self.sendQueued() def connectionLost(self, reason): log.clients("%s::connectionLost %s" % (self, reason.getErrorMessage())) self.connected = False def pauseProducing(self): self.paused = True def resumeProducing(self): self.paused = False self.sendQueued() def stopProducing(self): self.disconnect() def disconnect(self): if self.connected: self.transport.unregisterProducer() self.transport.loseConnection() self.connected = False def sendDatapoint(self, metric, datapoint): if self.paused: self.factory.enqueue(metric, datapoint) instrumentation.increment(self.queuedUntilReady) elif self.factory.hasQueuedDatapoints(): self.factory.enqueue(metric, datapoint) self.sendQueued() else: self._sendDatapoints([(metric, datapoint)]) def _sendDatapoints(self, datapoints): self.sendString(pickle.dumps(datapoints, protocol=-1)) instrumentation.increment(self.sent, len(datapoints)) self.factory.checkQueue() def sendQueued(self): while (not self.paused) and self.factory.hasQueuedDatapoints(): datapoints = self.factory.takeSomeFromQueue() self._sendDatapoints(datapoints) queueSize = self.factory.queueSize if (self.factory.queueFull.called and queueSize < SEND_QUEUE_LOW_WATERMARK): self.factory.queueHasSpace.callback(queueSize) def __str__(self): return 'CarbonClientProtocol(%s:%d:%s)' % (self.factory.destination) __repr__ = __str__ class CarbonClientFactory(ReconnectingClientFactory): maxDelay = 5 def __init__(self, destination): self.destination = destination self.destinationName = ('%s:%d:%s' % destination).replace('.', '_') self.host, self.port, self.carbon_instance = destination self.addr = (self.host, self.port) self.started = False # This factory maintains protocol state across reconnects self.queue = [] # including datapoints that still need to be sent self.connectedProtocol = None self.queueEmpty = Deferred() self.queueFull = Deferred() self.queueFull.addCallback(self.queueFullCallback) self.queueHasSpace = Deferred() self.queueHasSpace.addCallback(self.queueSpaceCallback) self.connectFailed = Deferred() self.connectionMade = Deferred() self.connectionLost = Deferred() # Define internal metric names self.attemptedRelays = 'destinations.%s.attemptedRelays' % self.destinationName self.fullQueueDrops = 'destinations.%s.fullQueueDrops' % self.destinationName self.queuedUntilConnected = 'destinations.%s.queuedUntilConnected' % self.destinationName def queueFullCallback(self, result): state.events.cacheFull() log.clients('%s send queue is full (%d datapoints)' % (self, result)) def queueSpaceCallback(self, result): if self.queueFull.called: log.clients('%s send queue has space available' % self.connectedProtocol) self.queueFull = Deferred() self.queueFull.addCallback(self.queueFullCallback) state.events.cacheSpaceAvailable() self.queueHasSpace = Deferred() self.queueHasSpace.addCallback(self.queueSpaceCallback) def buildProtocol(self, addr): self.connectedProtocol = CarbonClientProtocol() self.connectedProtocol.factory = self return self.connectedProtocol def startConnecting(self): # calling this startFactory yields recursion problems self.started = True self.connector = reactor.connectTCP(self.host, self.port, self) def stopConnecting(self): self.started = False self.stopTrying() if self.connectedProtocol and self.connectedProtocol.connected: return self.connectedProtocol.disconnect() @property def queueSize(self): return len(self.queue) def hasQueuedDatapoints(self): return bool(self.queue) def takeSomeFromQueue(self): datapoints = self.queue[:settings.MAX_DATAPOINTS_PER_MESSAGE] self.queue = self.queue[settings.MAX_DATAPOINTS_PER_MESSAGE:] return datapoints def checkQueue(self): if not self.queue: self.queueEmpty.callback(0) self.queueEmpty = Deferred() def enqueue(self, metric, datapoint): self.queue.append((metric, datapoint)) def sendDatapoint(self, metric, datapoint): instrumentation.increment(self.attemptedRelays) queueSize = self.queueSize if queueSize >= settings.MAX_QUEUE_SIZE: if not self.queueFull.called: self.queueFull.callback(queueSize) instrumentation.increment(self.fullQueueDrops) elif self.connectedProtocol: self.connectedProtocol.sendDatapoint(metric, datapoint) else: self.enqueue(metric, datapoint) instrumentation.increment(self.queuedUntilConnected) def startedConnecting(self, connector): log.clients("%s::startedConnecting (%s:%d)" % (self, connector.host, connector.port)) def clientConnectionLost(self, connector, reason): ReconnectingClientFactory.clientConnectionLost(self, connector, reason) log.clients("%s::clientConnectionLost (%s:%d) %s" % (self, connector.host, connector.port, reason.getErrorMessage())) self.connectedProtocol = None self.connectionLost.callback(0) self.connectionLost = Deferred() def clientConnectionFailed(self, connector, reason): ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) log.clients("%s::clientConnectionFailed (%s:%d) %s" % (self, connector.host, connector.port, reason.getErrorMessage())) self.connectFailed.callback(dict(connector=connector, reason=reason)) self.connectFailed = Deferred() def disconnect(self): self.queueEmpty.addCallback(lambda result: self.stopConnecting()) readyToStop = DeferredList( [self.connectionLost, self.connectFailed], fireOnOneCallback=True, fireOnOneErrback=True) self.checkQueue() # This can happen if the client is stopped before a connection is ever made if (not readyToStop.called) and (not self.started): readyToStop.callback(None) return readyToStop def __str__(self): return 'CarbonClientFactory(%s:%d:%s)' % self.destination __repr__ = __str__ class CarbonClientManager(Service): def __init__(self, router): self.router = router self.client_factories = {} # { destination : CarbonClientFactory() } def startService(self): Service.startService(self) for factory in self.client_factories.values(): if not factory.started: factory.startConnecting() def stopService(self): Service.stopService(self) self.stopAllClients() def startClient(self, destination): if destination in self.client_factories: return log.clients("connecting to carbon daemon at %s:%d:%s" % destination) self.router.addDestination(destination) factory = self.client_factories[destination] = CarbonClientFactory(destination) connectAttempted = DeferredList( [factory.connectionMade, factory.connectFailed], fireOnOneCallback=True, fireOnOneErrback=True) if self.running: factory.startConnecting() # this can trigger & replace connectFailed return connectAttempted def stopClient(self, destination): factory = self.client_factories.get(destination) if factory is None: return self.router.removeDestination(destination) stopCompleted = factory.disconnect() stopCompleted.addCallback(lambda result: self.disconnectClient(destination)) return stopCompleted def disconnectClient(self, destination): factory = self.client_factories.pop(destination) c = factory.connector if c and c.state == 'connecting' and not factory.hasQueuedDatapoints(): c.stopConnecting() def stopAllClients(self): deferreds = [] for destination in list(self.client_factories): deferreds.append( self.stopClient(destination) ) return DeferredList(deferreds) def sendDatapoint(self, metric, datapoint): for destination in self.router.getDestinations(metric): self.client_factories[destination].sendDatapoint(metric, datapoint) def __str__(self): return "<%s[%x]>" % (self.__class__.__name__, id(self)) carbon-0.9.12/lib/carbon/conf.py0000644000076400007640000004413412205151615016262 0ustar vagrantvagrant"""Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import os import sys import pwd import errno from os.path import join, dirname, normpath, exists, isdir from optparse import OptionParser from ConfigParser import ConfigParser import whisper from carbon import log from carbon.exceptions import CarbonConfigException from twisted.python import usage defaults = dict( USER="", MAX_CACHE_SIZE=float('inf'), MAX_UPDATES_PER_SECOND=500, MAX_CREATES_PER_MINUTE=float('inf'), LINE_RECEIVER_INTERFACE='0.0.0.0', LINE_RECEIVER_PORT=2003, ENABLE_UDP_LISTENER=False, UDP_RECEIVER_INTERFACE='0.0.0.0', UDP_RECEIVER_PORT=2003, PICKLE_RECEIVER_INTERFACE='0.0.0.0', PICKLE_RECEIVER_PORT=2004, CACHE_QUERY_INTERFACE='0.0.0.0', CACHE_QUERY_PORT=7002, LOG_UPDATES=True, LOG_CACHE_HITS=True, LOG_CACHE_QUEUE_SORTS=True, WHISPER_AUTOFLUSH=False, WHISPER_SPARSE_CREATE=False, WHISPER_FALLOCATE_CREATE=False, WHISPER_LOCK_WRITES=False, MAX_DATAPOINTS_PER_MESSAGE=500, MAX_AGGREGATION_INTERVALS=5, FORWARD_ALL=False, MAX_QUEUE_SIZE=1000, ENABLE_AMQP=False, AMQP_VERBOSE=False, BIND_PATTERNS=['#'], ENABLE_MANHOLE=False, MANHOLE_INTERFACE='127.0.0.1', MANHOLE_PORT=7222, MANHOLE_USER="", MANHOLE_PUBLIC_KEY="", RELAY_METHOD='rules', REPLICATION_FACTOR=1, DESTINATIONS=[], USE_FLOW_CONTROL=True, USE_INSECURE_UNPICKLER=False, USE_WHITELIST=False, CARBON_METRIC_PREFIX='carbon', CARBON_METRIC_INTERVAL=60, CACHE_WRITE_STRATEGY='sorted', WRITE_BACK_FREQUENCY=None, ENABLE_LOGROTATION=True, LOG_LISTENER_CONNECTIONS=True, ) def _process_alive(pid): if exists("/proc"): return exists("/proc/%d" % pid) else: try: os.kill(int(pid), 0) return True except OSError, err: return err.errno == errno.EPERM class OrderedConfigParser(ConfigParser): """Hacky workaround to ensure sections are always returned in the order they are defined in. Note that this does *not* make any guarantees about the order of options within a section or the order in which sections get written back to disk on write().""" _ordered_sections = [] def read(self, path): # Verifies a file exists *and* is readable if not os.access(path, os.R_OK): raise CarbonConfigException("Error: Missing config file or wrong perms on %s" % path) result = ConfigParser.read(self, path) sections = [] for line in open(path): line = line.strip() if line.startswith('[') and line.endswith(']'): sections.append(line[1:-1]) self._ordered_sections = sections return result def sections(self): return list(self._ordered_sections) # return a copy for safety class Settings(dict): __getattr__ = dict.__getitem__ def __init__(self): dict.__init__(self) self.update(defaults) def readFrom(self, path, section): parser = ConfigParser() if not parser.read(path): raise CarbonConfigException("Failed to read config file %s" % path) if not parser.has_section(section): return for key, value in parser.items(section): key = key.upper() # Detect type from defaults dict if key in defaults: valueType = type(defaults[key]) else: valueType = str if valueType is list: value = [v.strip() for v in value.split(',')] elif valueType is bool: value = parser.getboolean(section, key) else: # Attempt to figure out numeric types automatically try: value = int(value) except: try: value = float(value) except: pass self[key] = value settings = Settings() settings.update(defaults) class CarbonCacheOptions(usage.Options): optFlags = [ ["debug", "", "Run in debug mode."], ] optParameters = [ ["config", "c", None, "Use the given config file."], ["instance", "", "a", "Manage a specific carbon instance."], ["logdir", "", None, "Write logs to the given directory."], ["whitelist", "", None, "List of metric patterns to allow."], ["blacklist", "", None, "List of metric patterns to disallow."], ] def postOptions(self): global settings program = self.parent.subCommand # Use provided pidfile (if any) as default for configuration. If it's # set to 'twistd.pid', that means no value was provided and the default # was used. pidfile = self.parent["pidfile"] if pidfile.endswith("twistd.pid"): pidfile = None self["pidfile"] = pidfile # Enforce a default umask of '022' if none was set. if not self.parent.has_key("umask") or self.parent["umask"] is None: self.parent["umask"] = 022 # Read extra settings from the configuration file. program_settings = read_config(program, self) settings.update(program_settings) settings["program"] = program # Set process uid/gid by changing the parent config, if a user was # provided in the configuration file. if settings.USER: self.parent["uid"], self.parent["gid"] = ( pwd.getpwnam(settings.USER)[2:4]) # Set the pidfile in parent config to the value that was computed by # C{read_config}. self.parent["pidfile"] = settings["pidfile"] storage_schemas = join(settings["CONF_DIR"], "storage-schemas.conf") if not exists(storage_schemas): print "Error: missing required config %s" % storage_schemas sys.exit(1) if settings.WHISPER_AUTOFLUSH: log.msg("Enabling Whisper autoflush") whisper.AUTOFLUSH = True if settings.WHISPER_FALLOCATE_CREATE: if whisper.CAN_FALLOCATE: log.msg("Enabling Whisper fallocate support") else: log.err("WHISPER_FALLOCATE_CREATE is enabled but linking failed.") if settings.WHISPER_LOCK_WRITES: if whisper.CAN_LOCK: log.msg("Enabling Whisper file locking") whisper.LOCK = True else: log.err("WHISPER_LOCK_WRITES is enabled but import of fcntl module failed.") if settings.CACHE_WRITE_STRATEGY not in ('sorted', 'max', 'naive'): log.err("%s is not a valid value for CACHE_WRITE_STRATEGY, defaulting to %s" % (settings.CACHE_WRITE_STRATEGY, defaults['CACHE_WRITE_STRATEGY'])) else: log.msg("Using %s write strategy for cache" % settings.CACHE_WRITE_STRATEGY) if not "action" in self: self["action"] = "start" self.handleAction() # If we are not running in debug mode or non-daemon mode, then log to a # directory, otherwise log output will go to stdout. If parent options # are set to log to syslog, then use that instead. if not self["debug"]: if self.parent.get("syslog", None): log.logToSyslog(self.parent["prefix"]) elif not self.parent["nodaemon"]: logdir = settings.LOG_DIR if not isdir(logdir): os.makedirs(logdir) log.logToDir(logdir) if self["whitelist"] is None: self["whitelist"] = join(settings["CONF_DIR"], "whitelist.conf") settings["whitelist"] = self["whitelist"] if self["blacklist"] is None: self["blacklist"] = join(settings["CONF_DIR"], "blacklist.conf") settings["blacklist"] = self["blacklist"] def parseArgs(self, *action): """If an action was provided, store it for further processing.""" if len(action) == 1: self["action"] = action[0] def handleAction(self): """Handle extra argument for backwards-compatibility. * C{start} will simply do minimal pid checking and otherwise let twistd take over. * C{stop} will kill an existing running process if it matches the C{pidfile} contents. * C{status} will simply report if the process is up or not. """ action = self["action"] pidfile = self.parent["pidfile"] program = settings["program"] instance = self["instance"] if action == "stop": if not exists(pidfile): print "Pidfile %s does not exist" % pidfile raise SystemExit(0) pf = open(pidfile, 'r') try: pid = int(pf.read().strip()) pf.close() except: print "Could not read pidfile %s" % pidfile raise SystemExit(1) print "Sending kill signal to pid %d" % pid try: os.kill(pid, 15) except OSError, e: if e.errno == errno.ESRCH: print "No process with pid %d running" % pid else: raise raise SystemExit(0) elif action == "status": if not exists(pidfile): print "%s (instance %s) is not running" % (program, instance) raise SystemExit(1) pf = open(pidfile, "r") try: pid = int(pf.read().strip()) pf.close() except: print "Failed to read pid from %s" % pidfile raise SystemExit(1) if _process_alive(pid): print ("%s (instance %s) is running with pid %d" % (program, instance, pid)) raise SystemExit(0) else: print "%s (instance %s) is not running" % (program, instance) raise SystemExit(1) elif action == "start": if exists(pidfile): pf = open(pidfile, 'r') try: pid = int(pf.read().strip()) pf.close() except: print "Could not read pidfile %s" % pidfile raise SystemExit(1) if _process_alive(pid): print ("%s (instance %s) is already running with pid %d" % (program, instance, pid)) raise SystemExit(1) else: print "Removing stale pidfile %s" % pidfile try: os.unlink(pidfile) except: print "Could not remove pidfile %s" % pidfile print "Starting %s (instance %s)" % (program, instance) else: print "Invalid action '%s'" % action print "Valid actions: start stop status" raise SystemExit(1) class CarbonAggregatorOptions(CarbonCacheOptions): optParameters = [ ["rules", "", None, "Use the given aggregation rules file."], ["rewrite-rules", "", None, "Use the given rewrite rules file."], ] + CarbonCacheOptions.optParameters def postOptions(self): CarbonCacheOptions.postOptions(self) if self["rules"] is None: self["rules"] = join(settings["CONF_DIR"], "aggregation-rules.conf") settings["aggregation-rules"] = self["rules"] if self["rewrite-rules"] is None: self["rewrite-rules"] = join(settings["CONF_DIR"], "rewrite-rules.conf") settings["rewrite-rules"] = self["rewrite-rules"] class CarbonRelayOptions(CarbonCacheOptions): optParameters = [ ["rules", "", None, "Use the given relay rules file."], ["aggregation-rules", "", None, "Use the given aggregation rules file."], ] + CarbonCacheOptions.optParameters def postOptions(self): CarbonCacheOptions.postOptions(self) if self["rules"] is None: self["rules"] = join(settings["CONF_DIR"], "relay-rules.conf") settings["relay-rules"] = self["rules"] if self["aggregation-rules"] is None: self["aggregation-rules"] = join(settings["CONF_DIR"], "aggregation-rules.conf") settings["aggregation-rules"] = self["aggregation-rules"] if settings["RELAY_METHOD"] not in ("rules", "consistent-hashing", "aggregated-consistent-hashing"): print ("In carbon.conf, RELAY_METHOD must be either 'rules' or " "'consistent-hashing' or 'aggregated-consistent-hashing'. Invalid value: '%s'" % settings.RELAY_METHOD) sys.exit(1) def get_default_parser(usage="%prog [options] "): """Create a parser for command line options.""" parser = OptionParser(usage=usage) parser.add_option( "--debug", action="store_true", help="Run in the foreground, log to stdout") parser.add_option( "--profile", help="Record performance profile data to the given file") parser.add_option( "--pidfile", default=None, help="Write pid to the given file") parser.add_option( "--umask", default=None, help="Use the given umask when creating files") parser.add_option( "--config", default=None, help="Use the given config file") parser.add_option( "--whitelist", default=None, help="Use the given whitelist file") parser.add_option( "--blacklist", default=None, help="Use the given blacklist file") parser.add_option( "--logdir", default=None, help="Write logs in the given directory") parser.add_option( "--instance", default='a', help="Manage a specific carbon instance") return parser def get_parser(name): parser = get_default_parser() if name == "carbon-aggregator": parser.add_option( "--rules", default=None, help="Use the given aggregation rules file.") parser.add_option( "--rewrite-rules", default=None, help="Use the given rewrite rules file.") elif name == "carbon-relay": parser.add_option( "--rules", default=None, help="Use the given relay rules file.") return parser def parse_options(parser, args): """ Parse command line options and print usage message if no arguments were provided for the command. """ (options, args) = parser.parse_args(args) if not args: parser.print_usage() raise SystemExit(1) if args[0] not in ("start", "stop", "status"): parser.print_usage() raise SystemExit(1) return options, args def read_config(program, options, **kwargs): """ Read settings for 'program' from configuration file specified by 'options["config"]', with missing values provided by 'defaults'. """ settings = Settings() settings.update(defaults) # Initialize default values if not set yet. for name, value in kwargs.items(): settings.setdefault(name, value) graphite_root = kwargs.get("ROOT_DIR") if graphite_root is None: graphite_root = os.environ.get('GRAPHITE_ROOT') if graphite_root is None: raise CarbonConfigException("Either ROOT_DIR or GRAPHITE_ROOT " "needs to be provided.") # Default config directory to root-relative, unless overriden by the # 'GRAPHITE_CONF_DIR' environment variable. settings.setdefault("CONF_DIR", os.environ.get("GRAPHITE_CONF_DIR", join(graphite_root, "conf"))) if options["config"] is None: options["config"] = join(settings["CONF_DIR"], "carbon.conf") else: # Set 'CONF_DIR' to the parent directory of the 'carbon.conf' config # file. settings["CONF_DIR"] = dirname(normpath(options["config"])) # Storage directory can be overriden by the 'GRAPHITE_STORAGE_DIR' # environment variable. It defaults to a path relative to GRAPHITE_ROOT # for backwards compatibility though. settings.setdefault("STORAGE_DIR", os.environ.get("GRAPHITE_STORAGE_DIR", join(graphite_root, "storage"))) # By default, everything is written to subdirectories of the storage dir. settings.setdefault( "PID_DIR", settings["STORAGE_DIR"]) settings.setdefault( "LOG_DIR", join(settings["STORAGE_DIR"], "log", program)) settings.setdefault( "LOCAL_DATA_DIR", join(settings["STORAGE_DIR"], "whisper")) settings.setdefault( "WHITELISTS_DIR", join(settings["STORAGE_DIR"], "lists")) # Read configuration options from program-specific section. section = program[len("carbon-"):] config = options["config"] if not exists(config): raise CarbonConfigException("Error: missing required config %r" % config) settings.readFrom(config, section) settings.setdefault("instance", options["instance"]) # If a specific instance of the program is specified, augment the settings # with the instance-specific settings and provide sane defaults for # optional settings. if options["instance"]: settings.readFrom(config, "%s:%s" % (section, options["instance"])) settings["pidfile"] = ( options["pidfile"] or join(settings["PID_DIR"], "%s-%s.pid" % (program, options["instance"]))) settings["LOG_DIR"] = (options["logdir"] or join(settings["LOG_DIR"], "%s-%s" % (program, options["instance"]))) else: settings["pidfile"] = ( options["pidfile"] or join(settings["PID_DIR"], '%s.pid' % program)) settings["LOG_DIR"] = (options["logdir"] or settings["LOG_DIR"]) return settings carbon-0.9.12/lib/carbon/events.py0000644000076400007640000000264712205151615016644 0ustar vagrantvagrantfrom twisted.python.failure import Failure class Event: def __init__(self, name): self.name = name self.handlers = [] def addHandler(self, handler): if handler not in self.handlers: self.handlers.append(handler) def removeHandler(self, handler): if handler in self.handlers: self.handlers.remove(handler) def __call__(self, *args, **kwargs): for handler in self.handlers: try: handler(*args, **kwargs) except: log.err(None, "Exception in %s event handler: args=%s kwargs=%s" % (self.name, args, kwargs)) metricReceived = Event('metricReceived') metricGenerated = Event('metricGenerated') cacheFull = Event('cacheFull') cacheSpaceAvailable = Event('cacheSpaceAvailable') pauseReceivingMetrics = Event('pauseReceivingMetrics') resumeReceivingMetrics = Event('resumeReceivingMetrics') # Default handlers metricReceived.addHandler(lambda metric, datapoint: state.instrumentation.increment('metricsReceived')) cacheFull.addHandler(lambda: state.instrumentation.increment('cache.overflow')) cacheFull.addHandler(lambda: setattr(state, 'cacheTooFull', True)) cacheSpaceAvailable.addHandler(lambda: setattr(state, 'cacheTooFull', False)) pauseReceivingMetrics.addHandler(lambda: setattr(state, 'metricReceiversPaused', True)) resumeReceivingMetrics.addHandler(lambda: setattr(state, 'metricReceiversPaused', False)) # Avoid import circularities from carbon import log, state carbon-0.9.12/lib/carbon/exceptions.py0000644000076400007640000000014712204707552017520 0ustar vagrantvagrantclass CarbonConfigException(Exception): """Raised when a carbon daemon is improperly configured""" carbon-0.9.12/lib/carbon/hashing.py0000644000076400007640000000277612204707552016772 0ustar vagrantvagranttry: from hashlib import md5 except ImportError: from md5 import md5 import bisect class ConsistentHashRing: def __init__(self, nodes, replica_count=100): self.ring = [] self.nodes = set() self.replica_count = replica_count for node in nodes: self.add_node(node) def compute_ring_position(self, key): big_hash = md5( str(key) ).hexdigest() small_hash = int(big_hash[:4], 16) return small_hash def add_node(self, node): self.nodes.add(node) for i in range(self.replica_count): replica_key = "%s:%d" % (node, i) position = self.compute_ring_position(replica_key) entry = (position, node) bisect.insort(self.ring, entry) def remove_node(self, node): self.nodes.discard(node) self.ring = [entry for entry in self.ring if entry[1] != node] def get_node(self, key): assert self.ring node = None node_iter = self.get_nodes(key) node = node_iter.next() node_iter.close() return node def get_nodes(self, key): assert self.ring nodes = set() position = self.compute_ring_position(key) search_entry = (position, None) index = bisect.bisect_left(self.ring, search_entry) % len(self.ring) last_index = (index - 1) % len(self.ring) while len(nodes) < len(self.nodes) and index != last_index: next_entry = self.ring[index] (position, next_node) = next_entry if next_node not in nodes: nodes.add(next_node) yield next_node index = (index + 1) % len(self.ring) carbon-0.9.12/lib/carbon/instrumentation.py0000644000076400007640000001230212205170343020567 0ustar vagrantvagrantimport os import time import socket from resource import getrusage, RUSAGE_SELF from twisted.application.service import Service from twisted.internet.task import LoopingCall from carbon.conf import settings stats = {} HOSTNAME = socket.gethostname().replace('.','_') PAGESIZE = os.sysconf('SC_PAGESIZE') rusage = getrusage(RUSAGE_SELF) lastUsage = rusage.ru_utime + rusage.ru_stime lastUsageTime = time.time() # NOTE: Referencing settings in this *top level scope* will # give you *defaults* only. Probably not what you wanted. # TODO(chrismd) refactor the graphite metrics hierarchy to be cleaner, # more consistent, and make room for frontend metrics. #metric_prefix = "Graphite.backend.%(program)s.%(instance)s." % settings def increment(stat, increase=1): try: stats[stat] += increase except KeyError: stats[stat] = increase def append(stat, value): try: stats[stat].append(value) except KeyError: stats[stat] = [value] def getCpuUsage(): global lastUsage, lastUsageTime rusage = getrusage(RUSAGE_SELF) currentUsage = rusage.ru_utime + rusage.ru_stime currentTime = time.time() usageDiff = currentUsage - lastUsage timeDiff = currentTime - lastUsageTime if timeDiff == 0: #shouldn't be possible, but I've actually seen a ZeroDivisionError from this timeDiff = 0.000001 cpuUsagePercent = (usageDiff / timeDiff) * 100.0 lastUsage = currentUsage lastUsageTime = currentTime return cpuUsagePercent def getMemUsage(): rss_pages = int( open('/proc/self/statm').read().split()[1] ) return rss_pages * PAGESIZE def recordMetrics(): global lastUsage myStats = stats.copy() stats.clear() # cache metrics if settings.program == 'carbon-cache': record = cache_record updateTimes = myStats.get('updateTimes', []) committedPoints = myStats.get('committedPoints', 0) creates = myStats.get('creates', 0) errors = myStats.get('errors', 0) cacheQueries = myStats.get('cacheQueries', 0) cacheOverflow = myStats.get('cache.overflow', 0) # Calculate cache-data-structure-derived metrics prior to storing anything # in the cache itself -- which would otherwise affect said metrics. cache_size = cache.MetricCache.size cache_queues = len(cache.MetricCache) record('cache.size', cache_size) record('cache.queues', cache_queues) if updateTimes: avgUpdateTime = sum(updateTimes) / len(updateTimes) record('avgUpdateTime', avgUpdateTime) if committedPoints: pointsPerUpdate = float(committedPoints) / len(updateTimes) record('pointsPerUpdate', pointsPerUpdate) record('updateOperations', len(updateTimes)) record('committedPoints', committedPoints) record('creates', creates) record('errors', errors) record('cache.queries', cacheQueries) record('cache.overflow', cacheOverflow) # aggregator metrics elif settings.program == 'carbon-aggregator': record = aggregator_record record('allocatedBuffers', len(BufferManager)) record('bufferedDatapoints', sum([b.size for b in BufferManager.buffers.values()])) record('aggregateDatapointsSent', myStats.get('aggregateDatapointsSent', 0)) # relay metrics else: record = relay_record prefix = 'destinations.' relay_stats = [(k,v) for (k,v) in myStats.items() if k.startswith(prefix)] for stat_name, stat_value in relay_stats: record(stat_name, stat_value) # common metrics record('metricsReceived', myStats.get('metricsReceived', 0)) record('cpuUsage', getCpuUsage()) try: # This only works on Linux record('memUsage', getMemUsage()) except: pass def cache_record(metric, value): prefix = settings.CARBON_METRIC_PREFIX if settings.instance is None: fullMetric = '%s.agents.%s.%s' % (prefix, HOSTNAME, metric) else: fullMetric = '%s.agents.%s-%s.%s' % (prefix, HOSTNAME, settings.instance, metric) datapoint = (time.time(), value) cache.MetricCache.store(fullMetric, datapoint) def relay_record(metric, value): prefix = settings.CARBON_METRIC_PREFIX if settings.instance is None: fullMetric = '%s.relays.%s.%s' % (prefix, HOSTNAME, metric) else: fullMetric = '%s.relays.%s-%s.%s' % (prefix, HOSTNAME, settings.instance, metric) datapoint = (time.time(), value) events.metricGenerated(fullMetric, datapoint) def aggregator_record(metric, value): prefix = settings.CARBON_METRIC_PREFIX if settings.instance is None: fullMetric = '%s.aggregator.%s.%s' % (prefix, HOSTNAME, metric) else: fullMetric = '%s.aggregator.%s-%s.%s' % (prefix, HOSTNAME, settings.instance, metric) datapoint = (time.time(), value) events.metricGenerated(fullMetric, datapoint) class InstrumentationService(Service): def __init__(self): self.record_task = LoopingCall(recordMetrics) def startService(self): if settings.CARBON_METRIC_INTERVAL > 0: self.record_task.start(settings.CARBON_METRIC_INTERVAL, False) Service.startService(self) def stopService(self): if settings.CARBON_METRIC_INTERVAL > 0: self.record_task.stop() Service.stopService(self) # Avoid import circularities from carbon import state, events, cache from carbon.aggregator.buffers import BufferManager carbon-0.9.12/lib/carbon/log.py0000644000076400007640000000661412205151615016117 0ustar vagrantvagrantimport time from os.path import exists from sys import stdout, stderr from zope.interface import implements from twisted.python.log import startLoggingWithObserver, textFromEventDict, msg, err, ILogObserver from twisted.python.syslog import SyslogObserver from twisted.python.logfile import DailyLogFile class CarbonLogFile(DailyLogFile): """Overridden to support logrotate.d""" def __init__(self, *args, **kwargs): DailyLogFile.__init__(self, *args, **kwargs) # avoid circular dependencies from carbon.conf import settings self.enableRotation = settings.ENABLE_LOGROTATION def shouldRotate(self): if self.enableRotation: return DailyLogFile.shouldRotate(self) else: return False def write(self, data): if not self.enableRotation: if not exists(self.path): self.reopen() DailyLogFile.write(self, data) # Backport from twisted >= 10 def reopen(self): self.close() self._openFile() class CarbonLogObserver(object): implements(ILogObserver) def log_to_dir(self, logdir): self.logdir = logdir self.console_logfile = CarbonLogFile('console.log', logdir) self.custom_logs = {} self.observer = self.logdir_observer def log_to_syslog(self, prefix): observer = SyslogObserver(prefix).emit def syslog_observer(event): event["system"] = event.get("type", "console") observer(event) self.observer = syslog_observer def __call__(self, event): return self.observer(event) def stdout_observer(self, event): stdout.write(formatEvent(event, includeType=True) + '\n') stdout.flush() def logdir_observer(self, event): message = formatEvent(event) log_type = event.get('type') if log_type is not None and log_type not in self.custom_logs: self.custom_logs[log_type] = CarbonLogFile(log_type + '.log', self.logdir) logfile = self.custom_logs.get(log_type, self.console_logfile) logfile.write(message + '\n') logfile.flush() # Default to stdout observer = stdout_observer carbonLogObserver = CarbonLogObserver() def formatEvent(event, includeType=False): event['isError'] = 'failure' in event message = textFromEventDict(event) if includeType: typeTag = '[%s] ' % event.get('type', 'console') else: typeTag = '' timestamp = time.strftime("%d/%m/%Y %H:%M:%S") return "%s :: %s%s" % (timestamp, typeTag, message) logToDir = carbonLogObserver.log_to_dir logToSyslog = carbonLogObserver.log_to_syslog def logToStdout(): startLoggingWithObserver(carbonLogObserver) def cache(message, **context): context['type'] = 'cache' msg(message, **context) def clients(message, **context): context['type'] = 'clients' msg(message, **context) def creates(message, **context): context['type'] = 'creates' msg(message, **context) def updates(message, **context): context['type'] = 'updates' msg(message, **context) def listener(message, **context): context['type'] = 'listener' msg(message, **context) def relay(message, **context): context['type'] = 'relay' msg(message, **context) def aggregator(message, **context): context['type'] = 'aggregator' msg(message, **context) def query(message, **context): context['type'] = 'query' msg(message, **context) def debug(message, **context): if debugEnabled: msg(message, **context) debugEnabled = False def setDebugEnabled(enabled): global debugEnabled debugEnabled = enabled carbon-0.9.12/lib/carbon/management.py0000644000076400007640000000143612205151615017447 0ustar vagrantvagrantimport traceback import whisper from carbon import log from carbon.storage import getFilesystemPath def getMetadata(metric, key): if key != 'aggregationMethod': return dict(error="Unsupported metadata key \"%s\"" % key) wsp_path = getFilesystemPath(metric) try: value = whisper.info(wsp_path)['aggregationMethod'] return dict(value=value) except: log.err() return dict(error=traceback.format_exc()) def setMetadata(metric, key, value): if key != 'aggregationMethod': return dict(error="Unsupported metadata key \"%s\"" % key) wsp_path = getFilesystemPath(metric) try: old_value = whisper.setAggregationMethod(wsp_path, value) return dict(old_value=old_value, new_value=value) except: log.err() return dict(error=traceback.format_exc()) carbon-0.9.12/lib/carbon/manhole.py0000644000076400007640000000270312204707552016762 0ustar vagrantvagrantfrom twisted.cred import portal, checkers from twisted.conch.ssh import keys from twisted.conch.checkers import SSHPublicKeyDatabase from twisted.conch.manhole import Manhole from twisted.conch.manhole_ssh import TerminalRealm, ConchFactory from twisted.internet import reactor from carbon.conf import settings namespace = {} class PublicKeyChecker(SSHPublicKeyDatabase): def __init__(self, userKeys): self.userKeys = {} for username, keyData in userKeys.items(): self.userKeys[username] = keys.Key.fromString(data=keyData).blob() def checkKey(self, credentials): if credentials.username in self.userKeys: keyBlob = self.userKeys[credentials.username] return keyBlob == credentials.blob def createManholeListener(): sshRealm = TerminalRealm() sshRealm.chainedProtocolFactory.protocolFactory = lambda _: Manhole(namespace) # You can uncomment this if you're lazy and want insecure authentication instead # of setting up keys. #credChecker = checkers.InMemoryUsernamePasswordDatabaseDontUse(carbon='') userKeys = { settings.MANHOLE_USER : settings.MANHOLE_PUBLIC_KEY, } credChecker = PublicKeyChecker(userKeys) sshPortal = portal.Portal(sshRealm) sshPortal.registerChecker(credChecker) sessionFactory = ConchFactory(sshPortal) return sessionFactory def start(): sessionFactory = createManholeListener() reactor.listenTCP(settings.MANHOLE_PORT, sessionFactory, interface=settings.MANHOLE_INTERFACE) carbon-0.9.12/lib/carbon/protocols.py0000644000076400007640000001234112205170343017353 0ustar vagrantvagrantimport time from twisted.internet.protocol import DatagramProtocol from twisted.internet.error import ConnectionDone from twisted.protocols.basic import LineOnlyReceiver, Int32StringReceiver from carbon import log, events, state, management from carbon.conf import settings from carbon.regexlist import WhiteList, BlackList from carbon.util import pickle, get_unpickler class MetricReceiver: """ Base class for all metric receiving protocols, handles flow control events and connection state logging. """ def connectionMade(self): self.peerName = self.getPeerName() if settings.LOG_LISTENER_CONNECTIONS: log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerName)) if state.metricReceiversPaused: self.pauseReceiving() state.connectedMetricReceiverProtocols.add(self) events.pauseReceivingMetrics.addHandler(self.pauseReceiving) events.resumeReceivingMetrics.addHandler(self.resumeReceiving) def getPeerName(self): if hasattr(self.transport, 'getPeer'): peer = self.transport.getPeer() return "%s:%d" % (peer.host, peer.port) else: return "peer" def pauseReceiving(self): self.transport.pauseProducing() def resumeReceiving(self): self.transport.resumeProducing() def connectionLost(self, reason): if reason.check(ConnectionDone): if settings.LOG_LISTENER_CONNECTIONS: log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerName)) else: log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerName, reason.value)) state.connectedMetricReceiverProtocols.remove(self) events.pauseReceivingMetrics.removeHandler(self.pauseReceiving) events.resumeReceivingMetrics.removeHandler(self.resumeReceiving) def metricReceived(self, metric, datapoint): if BlackList and metric in BlackList: instrumentation.increment('blacklistMatches') return if WhiteList and metric not in WhiteList: instrumentation.increment('whitelistRejects') return if datapoint[1] != datapoint[1]: # filter out NaN values return if int(datapoint[0]) == -1: # use current time if none given datapoint = (time.time(), datapoint[1]) events.metricReceived(metric, datapoint) class MetricLineReceiver(MetricReceiver, LineOnlyReceiver): delimiter = '\n' def lineReceived(self, line): try: metric, value, timestamp = line.strip().split() datapoint = (float(timestamp), float(value)) except: log.listener('invalid line received from client %s, ignoring' % self.peerName) return self.metricReceived(metric, datapoint) class MetricDatagramReceiver(MetricReceiver, DatagramProtocol): def datagramReceived(self, data, (host, port)): for line in data.splitlines(): try: metric, value, timestamp = line.strip().split() datapoint = (float(timestamp), float(value)) self.metricReceived(metric, datapoint) except: log.listener('invalid line received from %s, ignoring' % host) class MetricPickleReceiver(MetricReceiver, Int32StringReceiver): MAX_LENGTH = 2 ** 20 def connectionMade(self): MetricReceiver.connectionMade(self) self.unpickler = get_unpickler(insecure=settings.USE_INSECURE_UNPICKLER) def stringReceived(self, data): try: datapoints = self.unpickler.loads(data) except: log.listener('invalid pickle received from %s, ignoring' % self.peerName) return for raw in datapoints: try: (metric, (value, timestamp)) = raw except Exception, e: log.listener('Error decoding pickle: %s' % e) try: datapoint = (float(value), float(timestamp)) # force proper types except: continue self.metricReceived(metric, datapoint) class CacheManagementHandler(Int32StringReceiver): def connectionMade(self): peer = self.transport.getPeer() self.peerAddr = "%s:%d" % (peer.host, peer.port) log.query("%s connected" % self.peerAddr) self.unpickler = get_unpickler(insecure=settings.USE_INSECURE_UNPICKLER) def connectionLost(self, reason): if reason.check(ConnectionDone): log.query("%s disconnected" % self.peerAddr) else: log.query("%s connection lost: %s" % (self.peerAddr, reason.value)) def stringReceived(self, rawRequest): request = self.unpickler.loads(rawRequest) if request['type'] == 'cache-query': metric = request['metric'] datapoints = MetricCache.get(metric, []) result = dict(datapoints=datapoints) if settings.LOG_CACHE_HITS is True: log.query('[%s] cache query for \"%s\" returned %d values' % (self.peerAddr, metric, len(datapoints))) instrumentation.increment('cacheQueries') elif request['type'] == 'get-metadata': result = management.getMetadata(request['metric'], request['key']) elif request['type'] == 'set-metadata': result = management.setMetadata(request['metric'], request['key'], request['value']) else: result = dict(error="Invalid request type \"%s\"" % request['type']) response = pickle.dumps(result, protocol=-1) self.sendString(response) # Avoid import circularities from carbon.cache import MetricCache from carbon import instrumentation carbon-0.9.12/lib/carbon/regexlist.py0000644000076400007640000000271312204707552017346 0ustar vagrantvagrantimport time import re import os.path from carbon import log from twisted.internet.task import LoopingCall class RegexList: """ Maintain a list of regex for matching whitelist and blacklist """ def __init__(self): self.regex_list = [] self.list_file = None self.read_task = LoopingCall(self.read_list) self.rules_last_read = 0.0 def read_from(self, list_file): self.list_file = list_file self.read_list() self.read_task.start(10, now=False) def read_list(self): # Clear rules and move on if file isn't there if not os.path.exists(self.list_file): self.regex_list = [] return try: mtime = os.path.getmtime(self.list_file) except: log.err("Failed to get mtime of %s" % self.list_file) return if mtime <= self.rules_last_read: return # Begin read new_regex_list = [] for line in open(self.list_file): pattern = line.strip() if line.startswith('#') or not pattern: continue try: new_regex_list.append(re.compile(pattern)) except: log.err("Failed to parse '%s' in '%s'. Ignoring line" % (pattern, self.list_file)) self.regex_list = new_regex_list self.rules_last_read = mtime def __contains__(self, value): for regex in self.regex_list: if regex.search(value): return True return False def __nonzero__(self): return bool(self.regex_list) WhiteList = RegexList() BlackList = RegexList() carbon-0.9.12/lib/carbon/relayrules.py0000644000076400007640000000425412204707552017531 0ustar vagrantvagrantimport re from carbon.conf import OrderedConfigParser from carbon.util import parseDestinations from carbon.exceptions import CarbonConfigException class RelayRule: def __init__(self, condition, destinations, continue_matching=False): self.condition = condition self.destinations = destinations self.continue_matching = continue_matching def matches(self, metric): return bool( self.condition(metric) ) def loadRelayRules(path): rules = [] parser = OrderedConfigParser() if not parser.read(path): raise CarbonConfigException("Could not read rules file %s" % path) defaultRule = None for section in parser.sections(): if not parser.has_option(section, 'destinations'): raise CarbonConfigException("Rules file %s section %s does not define a " "'destinations' list" % (path, section)) destination_strings = parser.get(section, 'destinations').split(',') destinations = parseDestinations(destination_strings) if parser.has_option(section, 'pattern'): if parser.has_option(section, 'default'): raise CarbonConfigException("Section %s contains both 'pattern' and " "'default'. You must use one or the other." % section) pattern = parser.get(section, 'pattern') regex = re.compile(pattern, re.I) continue_matching = False if parser.has_option(section, 'continue'): continue_matching = parser.getboolean(section, 'continue') rule = RelayRule(condition=regex.search, destinations=destinations, continue_matching=continue_matching) rules.append(rule) continue if parser.has_option(section, 'default'): if not parser.getboolean(section, 'default'): continue # just ignore default = false if defaultRule: raise CarbonConfigException("Only one default rule can be specified") defaultRule = RelayRule(condition=lambda metric: True, destinations=destinations) if not defaultRule: raise CarbonConfigException("No default rule defined. You must specify exactly one " "rule with 'default = true' instead of a pattern.") rules.append(defaultRule) return rules carbon-0.9.12/lib/carbon/rewrite.py0000644000076400007640000000337512204707552017026 0ustar vagrantvagrantimport time import re from os.path import exists, getmtime from twisted.internet.task import LoopingCall from carbon import log class RewriteRuleManager: def __init__(self): self.preRules = [] self.postRules = [] self.read_task = LoopingCall(self.read_rules) self.rules_last_read = 0.0 def clear(self): self.preRules = [] self.postRules = [] def read_from(self, rules_file): self.rules_file = rules_file self.read_rules() self.read_task.start(10, now=False) def read_rules(self): if not exists(self.rules_file): self.clear() return # Only read if the rules file has been modified try: mtime = getmtime(self.rules_file) except: log.err("Failed to get mtime of %s" % self.rules_file) return if mtime <= self.rules_last_read: return pre = [] post = [] section = None for line in open(self.rules_file): line = line.strip() if line.startswith('#') or not line: continue if line.startswith('[') and line.endswith(']'): section = line[1:-1].lower() else: pattern, replacement = line.split('=', 1) pattern, replacement = pattern.strip(), replacement.strip() rule = RewriteRule(pattern, replacement) if section == 'pre': pre.append(rule) elif section == 'post': post.append(rule) self.preRules = pre self.postRules = post self.rules_last_read = mtime class RewriteRule: def __init__(self, pattern, replacement): self.pattern = pattern self.replacement = replacement self.regex = re.compile(pattern) def apply(self, metric): return self.regex.sub(self.replacement, metric) # Ghetto singleton RewriteRuleManager = RewriteRuleManager() carbon-0.9.12/lib/carbon/routers.py0000644000076400007640000001003712205151615017033 0ustar vagrantvagrantimport imp from carbon.relayrules import loadRelayRules from carbon.hashing import ConsistentHashRing class DatapointRouter: "Interface for datapoint routing logic implementations" def addDestination(self, destination): "destination is a (host, port, instance) triple" def removeDestination(self, destination): "destination is a (host, port, instance) triple" def getDestinations(self, key): """Generate the destinations where the given routing key should map to. Only destinations which are configured (addDestination has been called for it) may be generated by this method.""" class RelayRulesRouter(DatapointRouter): def __init__(self, rules_path): self.rules_path = rules_path self.rules = loadRelayRules(rules_path) self.destinations = set() def addDestination(self, destination): self.destinations.add(destination) def removeDestination(self, destination): self.destinations.discard(destination) def getDestinations(self, key): for rule in self.rules: if rule.matches(key): for destination in rule.destinations: if destination in self.destinations: yield destination if not rule.continue_matching: return class ConsistentHashingRouter(DatapointRouter): def __init__(self, replication_factor=1): self.replication_factor = int(replication_factor) self.instance_ports = {} # { (server, instance) : port } self.ring = ConsistentHashRing([]) def addDestination(self, destination): (server, port, instance) = destination if (server, instance) in self.instance_ports: raise Exception("destination instance (%s, %s) already configured" % (server, instance)) self.instance_ports[(server, instance)] = port self.ring.add_node((server, instance)) def removeDestination(self, destination): (server, port, instance) = destination if (server, instance) not in self.instance_ports: raise Exception("destination instance (%s, %s) not configured" % (server, instance)) del self.instance_ports[(server, instance)] self.ring.remove_node((server, instance)) def getDestinations(self, metric): key = self.getKey(metric) for count,node in enumerate(self.ring.get_nodes(key)): if count == self.replication_factor: return (server, instance) = node port = self.instance_ports[ (server, instance) ] yield (server, port, instance) def getKey(self, metric): return metric def setKeyFunction(self, func): self.getKey = func def setKeyFunctionFromModule(self, keyfunc_spec): module_path, func_name = keyfunc_spec.rsplit(':', 1) module_file = open(module_path, 'U') description = ('.py', 'U', imp.PY_SOURCE) module = imp.load_module('keyfunc_module', module_file, module_path, description) keyfunc = getattr(module, func_name) self.setKeyFunction(keyfunc) class AggregatedConsistentHashingRouter(DatapointRouter): def __init__(self, agg_rules_manager, replication_factor=1): self.hash_router = ConsistentHashingRouter(replication_factor) self.agg_rules_manager = agg_rules_manager def addDestination(self, destination): self.hash_router.addDestination(destination) def removeDestination(self, destination): self.hash_router.removeDestination(destination) def getDestinations(self, key): # resolve metric to aggregate forms resolved_metrics = [] for rule in self.agg_rules_manager.rules: aggregate_metric = rule.get_aggregate_metric(key) if aggregate_metric is None: continue else: resolved_metrics.append(aggregate_metric) # if the metric will not be aggregated, send it raw # (will pass through aggregation) if len(resolved_metrics) == 0: resolved_metrics.append(key) # get consistent hashing destinations based on aggregate forms destinations = set() for resolved_metric in resolved_metrics: for destination in self.hash_router.getDestinations(resolved_metric): destinations.add(destination) for destination in destinations: yield destination carbon-0.9.12/lib/carbon/service.py0000644000076400007640000001704312205151615016774 0ustar vagrantvagrant"""Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" from os.path import exists from twisted.application.service import MultiService from twisted.application.internet import TCPServer, TCPClient, UDPServer from twisted.internet.protocol import ServerFactory from twisted.python.components import Componentized from twisted.python.log import ILogObserver # Attaching modules to the global state module simplifies import order hassles from carbon import util, state, events, instrumentation from carbon.log import carbonLogObserver from carbon.exceptions import CarbonConfigException state.events = events state.instrumentation = instrumentation class CarbonRootService(MultiService): """Root Service that properly configures twistd logging""" def setServiceParent(self, parent): MultiService.setServiceParent(self, parent) if isinstance(parent, Componentized): parent.setComponent(ILogObserver, carbonLogObserver) def createBaseService(config): from carbon.conf import settings from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver) root_service = CarbonRootService() root_service.setName(settings.program) use_amqp = settings.get("ENABLE_AMQP", False) if use_amqp: from carbon import amqp_listener amqp_host = settings.get("AMQP_HOST", "localhost") amqp_port = settings.get("AMQP_PORT", 5672) amqp_user = settings.get("AMQP_USER", "guest") amqp_password = settings.get("AMQP_PASSWORD", "guest") amqp_verbose = settings.get("AMQP_VERBOSE", False) amqp_vhost = settings.get("AMQP_VHOST", "/") amqp_spec = settings.get("AMQP_SPEC", None) amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite") for interface, port, protocol in ((settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT, MetricLineReceiver), (settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT, MetricPickleReceiver)): if port: factory = ServerFactory() factory.protocol = protocol service = TCPServer(int(port), factory, interface=interface) service.setServiceParent(root_service) if settings.ENABLE_UDP_LISTENER: service = UDPServer(int(settings.UDP_RECEIVER_PORT), MetricDatagramReceiver(), interface=settings.UDP_RECEIVER_INTERFACE) service.setServiceParent(root_service) if use_amqp: factory = amqp_listener.createAMQPListener( amqp_user, amqp_password, vhost=amqp_vhost, spec=amqp_spec, exchange_name=amqp_exchange_name, verbose=amqp_verbose) service = TCPClient(amqp_host, int(amqp_port), factory) service.setServiceParent(root_service) if settings.ENABLE_MANHOLE: from carbon import manhole factory = manhole.createManholeListener() service = TCPServer(int(settings.MANHOLE_PORT), factory, interface=settings.MANHOLE_INTERFACE) service.setServiceParent(root_service) if settings.USE_WHITELIST: from carbon.regexlist import WhiteList, BlackList WhiteList.read_from(settings["whitelist"]) BlackList.read_from(settings["blacklist"]) # Instantiate an instrumentation service that will record metrics about # this service. from carbon.instrumentation import InstrumentationService service = InstrumentationService() service.setServiceParent(root_service) return root_service def createCacheService(config): from carbon.cache import MetricCache from carbon.conf import settings from carbon.protocols import CacheManagementHandler # Configure application components events.metricReceived.addHandler(MetricCache.store) root_service = createBaseService(config) factory = ServerFactory() factory.protocol = CacheManagementHandler service = TCPServer(int(settings.CACHE_QUERY_PORT), factory, interface=settings.CACHE_QUERY_INTERFACE) service.setServiceParent(root_service) # have to import this *after* settings are defined from carbon.writer import WriterService service = WriterService() service.setServiceParent(root_service) if settings.USE_FLOW_CONTROL: events.cacheFull.addHandler(events.pauseReceivingMetrics) events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics) return root_service def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter() client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service def createRelayService(config): from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components if settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) elif settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter(settings.REPLICATION_FACTOR) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager RuleManager.read_from(settings["aggregation-rules"]) router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(client_manager.sendDatapoint) events.metricGenerated.addHandler(client_manager.sendDatapoint) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service carbon-0.9.12/lib/carbon/state.py0000644000076400007640000000031012204707552016447 0ustar vagrantvagrant__doc__ = """ This module exists for the purpose of tracking global state used across several modules. """ metricReceiversPaused = False cacheTooFull = False connectedMetricReceiverProtocols = set() carbon-0.9.12/lib/carbon/storage.py0000644000076400007640000001250012205151615016771 0ustar vagrantvagrant"""Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import os import re import whisper from os.path import join, exists, sep from carbon.conf import OrderedConfigParser, settings from carbon.util import pickle from carbon import log from carbon.exceptions import CarbonConfigException STORAGE_SCHEMAS_CONFIG = join(settings.CONF_DIR, 'storage-schemas.conf') STORAGE_AGGREGATION_CONFIG = join(settings.CONF_DIR, 'storage-aggregation.conf') STORAGE_LISTS_DIR = join(settings.CONF_DIR, 'lists') def getFilesystemPath(metric): metric_path = metric.replace('.', sep).lstrip(sep) + '.wsp' return join(settings.LOCAL_DATA_DIR, metric_path) class Schema: def test(self, metric): raise NotImplementedError() def matches(self, metric): return bool(self.test(metric)) class DefaultSchema(Schema): def __init__(self, name, archives): self.name = name self.archives = archives def test(self, metric): return True class PatternSchema(Schema): def __init__(self, name, pattern, archives): self.name = name self.pattern = pattern self.regex = re.compile(pattern) self.archives = archives def test(self, metric): return self.regex.search(metric) class ListSchema(Schema): def __init__(self, name, listName, archives): self.name = name self.listName = listName self.archives = archives self.path = join(settings.WHITELISTS_DIR, listName) if exists(self.path): self.mtime = os.stat(self.path).st_mtime fh = open(self.path, 'rb') self.members = pickle.load(fh) fh.close() else: self.mtime = 0 self.members = frozenset() def test(self, metric): if exists(self.path): current_mtime = os.stat(self.path).st_mtime if current_mtime > self.mtime: self.mtime = current_mtime fh = open(self.path, 'rb') self.members = pickle.load(fh) fh.close() return metric in self.members class Archive: def __init__(self, secondsPerPoint, points): self.secondsPerPoint = int(secondsPerPoint) self.points = int(points) def __str__(self): return "Archive = (Seconds per point: %d, Datapoints to save: %d)" % (self.secondsPerPoint, self.points) def getTuple(self): return (self.secondsPerPoint, self.points) @staticmethod def fromString(retentionDef): (secondsPerPoint, points) = whisper.parseRetentionDef(retentionDef) return Archive(secondsPerPoint, points) def loadStorageSchemas(): schemaList = [] config = OrderedConfigParser() config.read(STORAGE_SCHEMAS_CONFIG) for section in config.sections(): options = dict(config.items(section)) matchAll = options.get('match-all') pattern = options.get('pattern') listName = options.get('list') retentions = options['retentions'].split(',') archives = [Archive.fromString(s) for s in retentions] if matchAll: mySchema = DefaultSchema(section, archives) elif pattern: mySchema = PatternSchema(section, pattern, archives) elif listName: mySchema = ListSchema(section, listName, archives) archiveList = [a.getTuple() for a in archives] try: whisper.validateArchiveList(archiveList) schemaList.append(mySchema) except whisper.InvalidConfiguration, e: log.msg("Invalid schemas found in %s: %s" % (section, e)) schemaList.append(defaultSchema) return schemaList def loadAggregationSchemas(): # NOTE: This abuses the Schema classes above, and should probably be refactored. schemaList = [] config = OrderedConfigParser() try: config.read(STORAGE_AGGREGATION_CONFIG) except (IOError, CarbonConfigException): log.msg("%s not found, ignoring." % STORAGE_AGGREGATION_CONFIG) for section in config.sections(): options = dict(config.items(section)) matchAll = options.get('match-all') pattern = options.get('pattern') listName = options.get('list') xFilesFactor = options.get('xfilesfactor') aggregationMethod = options.get('aggregationmethod') try: if xFilesFactor is not None: xFilesFactor = float(xFilesFactor) assert 0 <= xFilesFactor <= 1 if aggregationMethod is not None: assert aggregationMethod in whisper.aggregationMethods except: log.msg("Invalid schemas found in %s." % section) continue archives = (xFilesFactor, aggregationMethod) if matchAll: mySchema = DefaultSchema(section, archives) elif pattern: mySchema = PatternSchema(section, pattern, archives) elif listName: mySchema = ListSchema(section, listName, archives) schemaList.append(mySchema) schemaList.append(defaultAggregation) return schemaList defaultArchive = Archive(60, 60 * 24 * 7) # default retention for unclassified data (7 days of minutely data) defaultSchema = DefaultSchema('default', [defaultArchive]) defaultAggregation = DefaultSchema('default', (None, None)) carbon-0.9.12/lib/carbon/util.py0000644000076400007640000001637112205151615016314 0ustar vagrantvagrantimport copy import os import pwd import sys from os.path import abspath, basename, dirname try: from cStringIO import StringIO except ImportError: from StringIO import StringIO try: import cPickle as pickle USING_CPICKLE = True except: import pickle USING_CPICKLE = False from time import sleep, time from twisted.python.util import initgroups from twisted.scripts.twistd import runApp from twisted.scripts._twistd_unix import daemonize daemonize = daemonize # Backwards compatibility def dropprivs(user): uid, gid = pwd.getpwnam(user)[2:4] initgroups(uid, gid) os.setregid(gid, gid) os.setreuid(uid, uid) return (uid, gid) def run_twistd_plugin(filename): from carbon.conf import get_parser from twisted.scripts.twistd import ServerOptions bin_dir = dirname(abspath(filename)) root_dir = dirname(bin_dir) os.environ.setdefault('GRAPHITE_ROOT', root_dir) program = basename(filename).split('.')[0] # First, parse command line options as the legacy carbon scripts used to # do. parser = get_parser(program) (options, args) = parser.parse_args() if not args: parser.print_usage() return # This isn't as evil as you might think __builtins__["instance"] = options.instance __builtins__["program"] = program # Then forward applicable options to either twistd or to the plugin itself. twistd_options = ["--no_save"] # If no reactor was selected yet, try to use the epoll reactor if # available. try: from twisted.internet import epollreactor twistd_options.append("--reactor=epoll") except: pass if options.debug: twistd_options.extend(["--nodaemon"]) if options.profile: twistd_options.append("--profile") if options.pidfile: twistd_options.extend(["--pidfile", options.pidfile]) if options.umask: twistd_options.extend(["--umask", options.umask]) # Now for the plugin-specific options. twistd_options.append(program) if options.debug: twistd_options.append("--debug") for option_name, option_value in vars(options).items(): if (option_value is not None and option_name not in ("debug", "profile", "pidfile", "umask")): twistd_options.extend(["--%s" % option_name.replace("_", "-"), option_value]) # Finally, append extra args so that twistd has a chance to process them. twistd_options.extend(args) config = ServerOptions() config.parseOptions(twistd_options) runApp(config) def parseDestinations(destination_strings): destinations = [] for dest_string in destination_strings: parts = dest_string.strip().split(':') if len(parts) == 2: server, port = parts instance = None elif len(parts) == 3: server, port, instance = parts else: raise ValueError("Invalid destination string \"%s\"" % dest_string) destinations.append((server, int(port), instance)) return destinations # This whole song & dance is due to pickle being insecure # yet performance critical for carbon. We leave the insecure # mode (which is faster) as an option (USE_INSECURE_UNPICKLER). # The SafeUnpickler classes were largely derived from # http://nadiana.com/python-pickle-insecure if USING_CPICKLE: class SafeUnpickler(object): PICKLE_SAFE = { 'copy_reg': set(['_reconstructor']), '__builtin__': set(['object']), } @classmethod def find_class(cls, module, name): if not module in cls.PICKLE_SAFE: raise pickle.UnpicklingError('Attempting to unpickle unsafe module %s' % module) __import__(module) mod = sys.modules[module] if not name in cls.PICKLE_SAFE[module]: raise pickle.UnpicklingError('Attempting to unpickle unsafe class %s' % name) return getattr(mod, name) @classmethod def loads(cls, pickle_string): pickle_obj = pickle.Unpickler(StringIO(pickle_string)) pickle_obj.find_global = cls.find_class return pickle_obj.load() else: class SafeUnpickler(pickle.Unpickler): PICKLE_SAFE = { 'copy_reg': set(['_reconstructor']), '__builtin__': set(['object']), } def find_class(self, module, name): if not module in self.PICKLE_SAFE: raise pickle.UnpicklingError('Attempting to unpickle unsafe module %s' % module) __import__(module) mod = sys.modules[module] if not name in self.PICKLE_SAFE[module]: raise pickle.UnpicklingError('Attempting to unpickle unsafe class %s' % name) return getattr(mod, name) @classmethod def loads(cls, pickle_string): return cls(StringIO(pickle_string)).load() def get_unpickler(insecure=False): if insecure: return pickle else: return SafeUnpickler class TokenBucket(object): '''This is a basic tokenbucket rate limiter implementation for use in enforcing various configurable rate limits''' def __init__(self, capacity, fill_rate): '''Capacity is the total number of tokens the bucket can hold, fill rate is the rate in tokens (or fractional tokens) to be added to the bucket per second.''' self.capacity = float(capacity) self._tokens = float(capacity) self.fill_rate = float(fill_rate) self.timestamp = time() def drain(self, cost, blocking=False): '''Given a number of tokens (or fractions) drain will return True and drain the number of tokens from the bucket if the capacity allows, otherwise we return false and leave the contents of the bucket.''' if cost <= self.tokens: self._tokens -= cost return True else: if blocking: tokens_needed = cost - self._tokens seconds_per_token = 1 / self.fill_rate seconds_left = seconds_per_token * self.fill_rate sleep(self.timestamp + seconds_left - time()) self._tokens -= cost return True return False @property def tokens(self): '''The tokens property will return the current number of tokens in the bucket.''' if self._tokens < self.capacity: now = time() delta = self.fill_rate * (now - self.timestamp) self._tokens = min(self.capacity, self._tokens + delta) self.timestamp = now return self._tokens class defaultdict(dict): def __init__(self, default_factory=None, *a, **kw): if (default_factory is not None and not hasattr(default_factory, '__call__')): raise TypeError('first argument must be callable') dict.__init__(self, *a, **kw) self.default_factory = default_factory def __getitem__(self, key): try: return dict.__getitem__(self, key) except KeyError: return self.__missing__(key) def __missing__(self, key): if self.default_factory is None: raise KeyError(key) self[key] = value = self.default_factory() return value def __reduce__(self): if self.default_factory is None: args = tuple() else: args = self.default_factory, return type(self), args, None, None, self.iteritems() def copy(self): return self.__copy__() def __copy__(self): return type(self)(self.default_factory, self) def __deepcopy__(self, memo): return type(self)(self.default_factory, copy.deepcopy(self.items())) def __repr__(self): return 'defaultdict(%s, %s)' % (self.default_factory, dict.__repr__(self)) carbon-0.9.12/lib/carbon/writer.py0000644000076400007640000001512612205151615016650 0ustar vagrantvagrant"""Copyright 2009 Chris Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.""" import os import time from os.path import exists, dirname import whisper from carbon import state from carbon.cache import MetricCache from carbon.storage import getFilesystemPath, loadStorageSchemas,\ loadAggregationSchemas from carbon.conf import settings from carbon import log, events, instrumentation from carbon.util import TokenBucket from twisted.internet import reactor from twisted.internet.task import LoopingCall from twisted.application.service import Service SCHEMAS = loadStorageSchemas() AGGREGATION_SCHEMAS = loadAggregationSchemas() CACHE_SIZE_LOW_WATERMARK = settings.MAX_CACHE_SIZE * 0.95 # Inititalize token buckets so that we can enforce rate limits on creates and # updates if the config wants them. CREATE_BUCKET = None UPDATE_BUCKET = None if settings.MAX_CREATES_PER_MINUTE != float('inf'): capacity = settings.MAX_CREATES_PER_MINUTE fill_rate = float(settings.MAX_CREATES_PER_MINUTE) / 60 CREATE_BUCKET = TokenBucket(capacity, fill_rate) if settings.MAX_UPDATES_PER_SECOND != float('inf'): capacity = settings.MAX_UPDATES_PER_SECOND fill_rate = settings.MAX_UPDATES_PER_SECOND UPDATE_BUCKET = TokenBucket(capacity, fill_rate) def optimalWriteOrder(): """Generates metrics with the most cached values first and applies a soft rate limit on new metrics""" while MetricCache: (metric, datapoints) = MetricCache.pop() if state.cacheTooFull and MetricCache.size < CACHE_SIZE_LOW_WATERMARK: events.cacheSpaceAvailable() dbFilePath = getFilesystemPath(metric) dbFileExists = exists(dbFilePath) if not dbFileExists and CREATE_BUCKET: # If our tokenbucket has enough tokens available to create a new metric # file then yield the metric data to complete that operation. Otherwise # we'll just drop the metric on the ground and move on to the next # metric. # XXX This behavior should probably be configurable to no tdrop metrics # when rate limitng unless our cache is too big or some other legit # reason. if CREATE_BUCKET.drain(1): yield (metric, datapoints, dbFilePath, dbFileExists) continue yield (metric, datapoints, dbFilePath, dbFileExists) def writeCachedDataPoints(): "Write datapoints until the MetricCache is completely empty" while MetricCache: dataWritten = False for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder(): dataWritten = True if not dbFileExists: archiveConfig = None xFilesFactor, aggregationMethod = None, None for schema in SCHEMAS: if schema.matches(metric): log.creates('new metric %s matched schema %s' % (metric, schema.name)) archiveConfig = [archive.getTuple() for archive in schema.archives] break for schema in AGGREGATION_SCHEMAS: if schema.matches(metric): log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name)) xFilesFactor, aggregationMethod = schema.archives break if not archiveConfig: raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric) dbDir = dirname(dbFilePath) try: if not exists(dbDir): os.makedirs(dbDir, 0755) except OSError, e: log.err("%s" % e) log.creates("creating database file %s (archive=%s xff=%s agg=%s)" % (dbFilePath, archiveConfig, xFilesFactor, aggregationMethod)) whisper.create( dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE) instrumentation.increment('creates') # If we've got a rate limit configured lets makes sure we enforce it if UPDATE_BUCKET: UPDATE_BUCKET.drain(1, blocking=True) try: t1 = time.time() whisper.update_many(dbFilePath, datapoints) updateTime = time.time() - t1 except: log.msg("Error writing to %s" % (dbFilePath)) log.err() instrumentation.increment('errors') else: pointCount = len(datapoints) instrumentation.increment('committedPoints', pointCount) instrumentation.append('updateTimes', updateTime) if settings.LOG_UPDATES: log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime)) # Avoid churning CPU when only new metrics are in the cache if not dataWritten: time.sleep(0.1) def writeForever(): while reactor.running: try: writeCachedDataPoints() except: log.err() time.sleep(1) # The writer thread only sleeps when the cache is empty or an error occurs def reloadStorageSchemas(): global SCHEMAS try: SCHEMAS = loadStorageSchemas() except: log.msg("Failed to reload storage SCHEMAS") log.err() def reloadAggregationSchemas(): global AGGREGATION_SCHEMAS try: AGGREGATION_SCHEMAS = loadAggregationSchemas() except: log.msg("Failed to reload aggregation SCHEMAS") log.err() def shutdownModifyUpdateSpeed(): try: settings.MAX_UPDATES_PER_SECOND = settings.MAX_UPDATES_PER_SECOND_ON_SHUTDOWN log.msg("Carbon shutting down. Changed the update rate to: " + str(settings.MAX_UPDATES_PER_SECOND_ON_SHUTDOWN)) except KeyError: log.msg("Carbon shutting down. Update rate not changed") class WriterService(Service): def __init__(self): self.storage_reload_task = LoopingCall(reloadStorageSchemas) self.aggregation_reload_task = LoopingCall(reloadAggregationSchemas) def startService(self): self.storage_reload_task.start(60, False) self.aggregation_reload_task.start(60, False) reactor.addSystemEventTrigger('before', 'shutdown', shutdownModifyUpdateSpeed) reactor.callInThread(writeForever) Service.startService(self) def stopService(self): self.storage_reload_task.stop() self.aggregation_reload_task.stop() Service.stopService(self) carbon-0.9.12/lib/twisted/0000755000076400007640000000000012205170400015165 5ustar vagrantvagrantcarbon-0.9.12/lib/twisted/plugins/0000755000076400007640000000000012205170401016647 5ustar vagrantvagrantcarbon-0.9.12/lib/twisted/plugins/carbon_aggregator_plugin.py0000644000076400007640000000124312204707552024261 0ustar vagrantvagrantfrom zope.interface import implements from twisted.plugin import IPlugin from twisted.application.service import IServiceMaker from carbon import service from carbon import conf class CarbonAggregatorServiceMaker(object): implements(IServiceMaker, IPlugin) tapname = "carbon-aggregator" description = "Aggregate stats for graphite." options = conf.CarbonAggregatorOptions def makeService(self, options): """ Construct a C{carbon-aggregator} service. """ return service.createAggregatorService(options) # Now construct an object which *provides* the relevant interfaces serviceMaker = CarbonAggregatorServiceMaker() carbon-0.9.12/lib/twisted/plugins/carbon_cache_plugin.py0000644000076400007640000000120312204707552023176 0ustar vagrantvagrantfrom zope.interface import implements from twisted.plugin import IPlugin from twisted.application.service import IServiceMaker from carbon import service from carbon import conf class CarbonCacheServiceMaker(object): implements(IServiceMaker, IPlugin) tapname = "carbon-cache" description = "Collect stats for graphite." options = conf.CarbonCacheOptions def makeService(self, options): """ Construct a C{carbon-cache} service. """ return service.createCacheService(options) # Now construct an object which *provides* the relevant interfaces serviceMaker = CarbonCacheServiceMaker() carbon-0.9.12/lib/twisted/plugins/carbon_relay_plugin.py0000644000076400007640000000120112204707552023245 0ustar vagrantvagrantfrom zope.interface import implements from twisted.plugin import IPlugin from twisted.application.service import IServiceMaker from carbon import service from carbon import conf class CarbonRelayServiceMaker(object): implements(IServiceMaker, IPlugin) tapname = "carbon-relay" description = "Relay stats for graphite." options = conf.CarbonRelayOptions def makeService(self, options): """ Construct a C{carbon-relay} service. """ return service.createRelayService(options) # Now construct an object which *provides* the relevant interfaces serviceMaker = CarbonRelayServiceMaker() carbon-0.9.12/LICENSE0000644000076400007640000002613612022301500013743 0ustar vagrantvagrant Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. carbon-0.9.12/MANIFEST.in0000644000076400007640000000021512204707552014505 0ustar vagrantvagrantrecursive-include conf/ * recursive-include distro/ * exclude conf/*.conf include LICENSE include lib/carbon/amqp0-8.xml include MANIFEST.in carbon-0.9.12/PKG-INFO0000644000076400007640000000044312205170401014033 0ustar vagrantvagrantMetadata-Version: 1.0 Name: carbon Version: 0.9.12 Summary: Backend data caching and persistence daemon for Graphite Home-page: http://graphite-project.github.com Author: Chris Davis Author-email: chrismd@gmail.com License: Apache Software License 2.0 Description: UNKNOWN Platform: UNKNOWN carbon-0.9.12/setup.cfg0000644000076400007640000000024712022301500014552 0ustar vagrantvagrant[install] prefix = /opt/graphite install-lib = %(prefix)s/lib [bdist_rpm] requires = python-twisted whisper post-install = distro/redhat/misc/postinstall carbon-0.9.12/setup.py0000644000076400007640000000247112205170174014462 0ustar vagrantvagrant#!/usr/bin/env python import os import platform from glob import glob if os.environ.get('USE_SETUPTOOLS'): from setuptools import setup setup_kwargs = dict(zip_safe=0) else: from distutils.core import setup setup_kwargs = dict() storage_dirs = [ ('storage/whisper',[]), ('storage/lists',[]), ('storage/log',[]), ('storage/rrd',[]) ] conf_files = [ ('conf', glob('conf/*.example')) ] install_files = storage_dirs + conf_files # If we are building on RedHat, let's use the redhat init scripts. if platform.dist()[0] == 'redhat': init_scripts = [ ('/etc/init.d', ['distro/redhat/init.d/carbon-cache', 'distro/redhat/init.d/carbon-relay', 'distro/redhat/init.d/carbon-aggregator']) ] install_files += init_scripts setup( name='carbon', version='0.9.12', url='http://graphite-project.github.com', author='Chris Davis', author_email='chrismd@gmail.com', license='Apache Software License 2.0', description='Backend data caching and persistence daemon for Graphite', packages=['carbon', 'carbon.aggregator', 'twisted.plugins'], package_dir={'' : 'lib'}, scripts=glob('bin/*'), package_data={ 'carbon' : ['*.xml'] }, data_files=install_files, install_requires=['twisted', 'txamqp'], **setup_kwargs )