Commit 37b007fb authored by Jonas Genannt's avatar Jonas Genannt

Imported Upstream version 0.9.12

parent 1816fc82
Metadata-Version: 1.0
Name: carbon
Version: 0.9.10
Version: 0.9.12
Summary: Backend data caching and persistence daemon for Graphite
Home-page: https://launchpad.net/graphite
Home-page: http://graphite-project.github.com
Author: Chris Davis
Author-email: chrismd@gmail.com
License: Apache Software License 2.0
......
......@@ -14,17 +14,21 @@ See the License for the specific language governing permissions and
limitations under the License."""
import sys
from os.path import dirname, join, abspath
import os.path
# Figure out where we're installed
BIN_DIR = dirname(abspath(__file__))
ROOT_DIR = dirname(BIN_DIR)
BIN_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(BIN_DIR)
# Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from
# source.
LIB_DIR = join(ROOT_DIR, 'lib')
LIB_DIR = os.path.join(ROOT_DIR, "lib")
sys.path.insert(0, LIB_DIR)
from carbon.util import run_twistd_plugin
from carbon.exceptions import CarbonConfigException
run_twistd_plugin(__file__)
try:
run_twistd_plugin(__file__)
except CarbonConfigException, exc:
raise SystemExit(str(exc))
......@@ -14,17 +14,21 @@ See the License for the specific language governing permissions and
limitations under the License."""
import sys
from os.path import dirname, join, abspath
import os.path
# Figure out where we're installed
BIN_DIR = dirname(abspath(__file__))
ROOT_DIR = dirname(BIN_DIR)
BIN_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(BIN_DIR)
# Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from
# source.
LIB_DIR = join(ROOT_DIR, 'lib')
LIB_DIR = os.path.join(ROOT_DIR, "lib")
sys.path.insert(0, LIB_DIR)
from carbon.util import run_twistd_plugin
from carbon.exceptions import CarbonConfigException
run_twistd_plugin(__file__)
try:
run_twistd_plugin(__file__)
except CarbonConfigException, exc:
raise SystemExit(str(exc))
......@@ -14,17 +14,21 @@ See the License for the specific language governing permissions and
limitations under the License."""
import sys
from os.path import dirname, join, abspath
import os.path
# Figure out where we're installed
BIN_DIR = dirname(abspath(__file__))
ROOT_DIR = dirname(BIN_DIR)
BIN_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = os.path.dirname(BIN_DIR)
# Make sure that carbon's 'lib' dir is in the $PYTHONPATH if we're running from
# source.
LIB_DIR = join(ROOT_DIR, 'lib')
LIB_DIR = os.path.join(ROOT_DIR, "lib")
sys.path.insert(0, LIB_DIR)
from carbon.util import run_twistd_plugin
from carbon.exceptions import CarbonConfigException
run_twistd_plugin(__file__)
try:
run_twistd_plugin(__file__)
except CarbonConfigException, exc:
raise SystemExit(str(exc))
......@@ -27,8 +27,7 @@ else:
config_parser = ConfigParser()
if not config_parser.read(SCHEMAS_FILE):
print "Error: Couldn't read config file: %s" % SCHEMAS_FILE
sys.exit(1)
raise SystemExit("Error: Couldn't read config file: %s" % SCHEMAS_FILE)
errors_found = 0
......@@ -62,9 +61,6 @@ for section in config_parser.sections():
print " OK"
if errors_found:
print
print "Storage-schemas configuration '%s' failed validation" % SCHEMAS_FILE
sys.exit(1)
raise SystemExit( "Storage-schemas configuration '%s' failed validation" % SCHEMAS_FILE)
print
print "Storage-schemas configuration '%s' is valid" % SCHEMAS_FILE
......@@ -26,4 +26,10 @@
# aggregate metric 'prod.applications.apache.all.requests' would be calculated
# by summing their values.
#
# Template components such as <env> will match everything up to the next dot.
# To match metric multiple components including the dots, use <<metric>> in the
# input template:
#
# <env>.applications.<app>.all.<app_metric> (60) = sum <env>.applications.<app>.*.<<app_metric>>
#
# Note that any time this file is modified, it will be re-read automatically.
......@@ -30,10 +30,16 @@
#
#LOCAL_DATA_DIR = /opt/graphite/storage/whisper/
# Enable daily log rotation. If disabled, a kill -HUP can be used after a manual rotate
ENABLE_LOGROTATION = True
# Specify the user to drop privileges to
# If this is blank carbon runs as the user that invokes it
# This user must have write access to the local data directory
USER =
#
# NOTE: The above settings must be set under [relay] and [aggregator]
# to take effect for those daemons as well
# Limit the size of the cache to avoid swapping or becoming CPU bound.
# Sorts and serving cache queries gets more expensive as the cache grows.
......@@ -47,6 +53,12 @@ MAX_CACHE_SIZE = inf
# take effect and increase the overall throughput accordingly.
MAX_UPDATES_PER_SECOND = 500
# If defined, this changes the MAX_UPDATES_PER_SECOND in Carbon when a
# stop/shutdown is initiated. This helps when MAX_UPDATES_PER_SECOND is
# relatively low and carbon has cached a lot of updates; it enables the carbon
# daemon to shutdown more quickly.
# MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000
# Softly limits the number of whisper files that get created each minute.
# Setting this value low (like at 50) is a good way to ensure your graphite
# system will not be adversely impacted when a bunch of new metrics are
......@@ -69,6 +81,9 @@ UDP_RECEIVER_PORT = 2003
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2004
# Set to false to disable logging of successful connections
LOG_LISTENER_CONNECTIONS = True
# Per security concerns outlined in Bug #817247 the pickle receiver
# will use a more secure and slightly less efficient unpickler.
# Set this to True to revert to the old-fashioned insecure unpickler.
......@@ -83,9 +98,36 @@ CACHE_QUERY_PORT = 7002
# data until the cache size falls below 95% MAX_CACHE_SIZE.
USE_FLOW_CONTROL = True
# By default, carbon-cache will log every whisper update. This can be excessive and
# By default, carbon-cache will log every whisper update and cache hit. This can be excessive and
# degrade performance if logging on the same volume as the whisper data is stored.
LOG_UPDATES = False
LOG_CACHE_HITS = False
LOG_CACHE_QUEUE_SORTS = True
# The thread that writes metrics to disk can use on of the following strategies
# determining the order in which metrics are removed from cache and flushed to
# disk. The default option preserves the same behavior as has been historically
# available in version 0.9.10.
#
# sorted - All metrics in the cache will be counted and an ordered list of
# them will be sorted according to the number of datapoints in the cache at the
# moment of the list's creation. Metrics will then be flushed from the cache to
# disk in that order.
#
# max - The writer thread will always pop and flush the metric from cache
# that has the most datapoints. This will give a strong flush preference to
# frequently updated metrics and will also reduce random file-io. Infrequently
# updated metrics may only ever be persisted to disk at daemon shutdown if
# there are a large number of metrics which receive very frequent updates OR if
# disk i/o is very slow.
#
# naive - Metrics will be flushed from the cache to disk in an unordered
# fashion. This strategy may be desirable in situations where the storage for
# whisper files is solid state, CPU resources are very limited or deference to
# the OS's i/o scheduler is expected to compensate for the random write
# pattern.
#
CACHE_WRITE_STRATEGY = sorted
# On some systems it is desirable for whisper to write synchronously.
# Set this option to True if you'd like to try this. Basically it will
......@@ -100,6 +142,14 @@ WHISPER_AUTOFLUSH = False
# depending on the underlying storage configuration.
# WHISPER_SPARSE_CREATE = False
# Only beneficial on linux filesystems that support the fallocate system call.
# It maintains the benefits of contiguous reads/writes, but with a potentially
# much faster creation speed, by allowing the kernel to handle the block
# allocation and zero-ing. Enabling this option may allow a large increase of
# MAX_CREATES_PER_MINUTE. If enabled on an OS or filesystem that is unsupported
# this option will gracefully fallback to standard POSIX file access methods.
WHISPER_FALLOCATE_CREATE = True
# Enabling this option will cause Whisper to lock each Whisper file it writes
# to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when
# multiple carbon-cache daemons are writing to the same files
......@@ -170,14 +220,27 @@ LINE_RECEIVER_PORT = 2013
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2014
# To use consistent hashing instead of the user defined relay-rules.conf,
# change this to:
# RELAY_METHOD = consistent-hashing
# Set to false to disable logging of successful connections
LOG_LISTENER_CONNECTIONS = True
# Carbon-relay has several options for metric routing controlled by RELAY_METHOD
#
# Use relay-rules.conf to route metrics to destinations based on pattern rules
#RELAY_METHOD = rules
#
# Use consistent-hashing for even distribution of metrics between destinations
#RELAY_METHOD = consistent-hashing
#
# Use consistent-hashing but take into account an aggregation-rules.conf shared
# by downstream carbon-aggregator daemons. This will ensure that all metrics
# that map to a given aggregation rule are sent to the same carbon-aggregator
# instance.
# Enable this for carbon-relays that send to a group of carbon-aggregators
#RELAY_METHOD = aggregated-consistent-hashing
RELAY_METHOD = rules
# If you use consistent-hashing you may want to add redundancy
# of your data by replicating every datapoint to more than
# one machine.
# If you use consistent-hashing you can add redundancy by replicating every
# datapoint to more than one machine.
REPLICATION_FACTOR = 1
# This is a list of carbon daemons we will send any relayed or
......@@ -228,6 +291,14 @@ LINE_RECEIVER_PORT = 2023
PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2024
# Set to false to disable logging of successful connections
LOG_LISTENER_CONNECTIONS = True
# If set true, metric received will be forwarded to DESTINATIONS in addition to
# the output of the aggregation rules. If set false the carbon-aggregator will
# only ever send the output of aggregation.
FORWARD_ALL = True
# This is a list of carbon daemons we will send any relayed or
# generated metrics to. The default provided would send to a single
# carbon-cache instance on the default port. However if you
......@@ -268,6 +339,14 @@ MAX_DATAPOINTS_PER_MESSAGE = 500
# the past MAX_AGGREGATION_INTERVALS * intervalSize seconds.
MAX_AGGREGATION_INTERVALS = 5
# By default (WRITE_BACK_FREQUENCY = 0), carbon-aggregator will write back
# aggregated data points once every rule.frequency seconds, on a per-rule basis.
# Set this (WRITE_BACK_FREQUENCY = N) to write back all aggregated data points
# every N seconds, independent of rule frequency. This is useful, for example,
# to be able to query partially aggregated metrics from carbon-cache without
# having to first wait rule.frequency seconds.
# WRITE_BACK_FREQUENCY = 0
# Set this to True to enable whitelisting and blacklisting of metrics in
# CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or
# empty, all metrics will pass through
......
#!/bin/bash
# chkconfig: - 25 75
# description: carbon-aggregator
# processname: carbon-aggregator
export PYTHONPATH="$GRAPHITE_DIR/lib:$PYTHONPATH"
# Source function library.
if [ -e /etc/rc.d/init.d/functions ]; then
. /etc/rc.d/init.d/functions;
fi;
CARBON_DAEMON="aggregator"
GRAPHITE_DIR="/opt/graphite"
INSTANCES=`grep "^\[${CARBON_DAEMON}" ${GRAPHITE_DIR}/conf/carbon.conf | cut -d \[ -f 2 | cut -d \] -f 1 | cut -d : -f 2`
function die {
echo $1
exit 1
}
start(){
cd $GRAPHITE_DIR;
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
echo "Starting carbon-${CARBON_DAEMON}:${INSTANCE}..."
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} start;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
stop(){
cd $GRAPHITE_DIR
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
echo "Stopping carbon-${CARBON_DAEMON}:${INSTANCE}..."
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} stop
if [ `sleep 3; /usr/bin/pgrep -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}" | /usr/bin/wc -l` -gt 0 ]; then
echo "Carbon did not stop yet. Sleeping longer, then force killing it...";
sleep 20;
/usr/bin/pkill -9 -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}";
fi;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
status(){
cd $GRAPHITE_DIR;
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} status;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status
;;
restart|reload)
stop
start
;;
*)
echo $"Usage: $0 {start|stop|restart|status}"
exit 1
esac
#!/bin/bash
# chkconfig: - 25 75
# description: carbon-cache
# processname: carbon-cache
export PYTHONPATH="$GRAPHITE_DIR/lib:$PYTHONPATH"
# Source function library.
if [ -e /etc/rc.d/init.d/functions ]; then
. /etc/rc.d/init.d/functions;
fi;
CARBON_DAEMON="cache"
GRAPHITE_DIR="/opt/graphite"
INSTANCES=`grep "^\[${CARBON_DAEMON}" ${GRAPHITE_DIR}/conf/carbon.conf | cut -d \[ -f 2 | cut -d \] -f 1 | cut -d : -f 2`
function die {
echo $1
exit 1
}
start(){
cd $GRAPHITE_DIR;
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
echo "Starting carbon-${CARBON_DAEMON}:${INSTANCE}..."
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} start;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
stop(){
cd $GRAPHITE_DIR
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
echo "Stopping carbon-${CARBON_DAEMON}:${INSTANCE}..."
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} stop
if [ `sleep 3; /usr/bin/pgrep -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}" | /usr/bin/wc -l` -gt 0 ]; then
echo "Carbon did not stop yet. Sleeping longer, then force killing it...";
sleep 20;
/usr/bin/pkill -9 -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}";
fi;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
status(){
cd $GRAPHITE_DIR;
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} status;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status
;;
restart|reload)
stop
start
;;
*)
echo $"Usage: $0 {start|stop|restart|status}"
exit 1
esac
#!/bin/bash
# chkconfig: - 25 75
# description: carbon-relay
# processname: carbon-relay
export PYTHONPATH="$GRAPHITE_DIR/lib:$PYTHONPATH"
# Source function library.
if [ -e /etc/rc.d/init.d/functions ]; then
. /etc/rc.d/init.d/functions;
fi;
CARBON_DAEMON="relay"
GRAPHITE_DIR="/opt/graphite"
INSTANCES=`grep "^\[${CARBON_DAEMON}" ${GRAPHITE_DIR}/conf/carbon.conf | cut -d \[ -f 2 | cut -d \] -f 1 | cut -d : -f 2`
function die {
echo $1
exit 1
}
start(){
cd $GRAPHITE_DIR;
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
echo "Starting carbon-${CARBON_DAEMON}:${INSTANCE}..."
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} start;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
stop(){
cd $GRAPHITE_DIR
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
echo "Stopping carbon-${CARBON_DAEMON}:${INSTANCE}..."
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} stop
if [ `sleep 3; /usr/bin/pgrep -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}" | /usr/bin/wc -l` -gt 0 ]; then
echo "Carbon did not stop yet. Sleeping longer, then force killing it...";
sleep 20;
/usr/bin/pkill -9 -f "carbon-${CARBON_DAEMON}.py --instance=${INSTANCE}";
fi;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
status(){
cd $GRAPHITE_DIR;
for INSTANCE in ${INSTANCES}; do
if [ "${INSTANCE}" == "${CARBON_DAEMON}" ]; then
INSTANCE="a";
fi;
bin/carbon-${CARBON_DAEMON}.py --instance=${INSTANCE} status;
if [ $? -eq 0 ]; then
echo_success
else
echo_failure
fi;
echo ""
done;
}
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status
;;
restart|reload)
stop
start
;;
*)
echo $"Usage: $0 {start|stop|restart|status}"
exit 1
esac
......@@ -51,7 +51,7 @@ class MetricBuffer:
self.aggregation_frequency = int(frequency)
self.aggregation_func = func
self.compute_task = LoopingCall(self.compute_value)
self.compute_task.start(frequency, now=False)
self.compute_task.start(settings['WRITE_BACK_FREQUENCY'] or frequency, now=False)
self.configured = True
def compute_value(self):
......@@ -69,6 +69,10 @@ class MetricBuffer:
if buffer.interval < age_threshold:
del self.interval_buffers[buffer.interval]
if not self.interval_buffers:
self.close()
self.configured = False
del BufferManager.buffers[self.metric_path]
def close(self):
if self.compute_task and self.compute_task.running:
......
from carbon.instrumentation import increment
from carbon.aggregator.rules import RuleManager
from carbon.aggregator.buffers import BufferManager
from carbon.conf import settings
from carbon.rewrite import RewriteRuleManager
from carbon import events
......@@ -31,5 +32,5 @@ def process(metric, datapoint):
for rule in RewriteRuleManager.postRules:
metric = rule.apply(metric)
if metric not in aggregate_metrics:
if settings['FORWARD_ALL'] and metric not in aggregate_metrics:
events.metricGenerated(metric, datapoint)
......@@ -139,6 +139,8 @@ def avg(values):
AGGREGATION_METHODS = {
'sum' : sum,
'avg' : avg,
'min' : min,
'max' : max,
}
# Importable singleton
......
......@@ -35,7 +35,8 @@ import os
import socket
from optparse import OptionParser
from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
from twisted.internet.defer import deferredGenerator, waitForDeferred
from twisted.internet import reactor
from twisted.internet.protocol import ReconnectingClientFactory
from txamqp.protocol import AMQClient
......@@ -62,44 +63,65 @@ class AMQPGraphiteProtocol(AMQClient):
consumer_tag = "graphite_consumer"
@inlineCallbacks
@deferredGenerator
def connectionMade(self):
yield AMQClient.connectionMade(self)
AMQClient.connectionMade(self)
log.listener("New AMQP connection made")
yield self.setup()
yield self.receive_loop()
self.setup()
wfd = waitForDeferred(self.receive_loop())
yield wfd
@inlineCallbacks
@deferredGenerator
def setup(self):
exchange = self.factory.exchange_name
yield self.authenticate(self.factory.username, self.factory.password)
chan = yield self.channel(1)
yield chan.channel_open()
d = self.authenticate(self.factory.username, self.factory.password)
wfd = waitForDeferred(d)
yield wfd
wfd = waitForDeferred(self.channel(1))
yield wfd
chan = wfd.getResult()
wfd = waitForDeferred(chan.channel_open())
yield wfd
# declare the exchange and queue
yield chan.exchange_declare(exchange=exchange, type="topic",
durable=True, auto_delete=False)
d = chan.exchange_declare(exchange=exchange, type="topic",
durable=True, auto_delete=False)
wfd = waitForDeferred(d)
yield wfd
# we use a private queue to avoid conflicting with existing bindings
reply = yield chan.queue_declare(exclusive=True)
wfd = waitForDeferred(chan.queue_declare(exclusive=True))
yield wfd
reply = wfd.getResult()
my_queue = reply.queue
# bind each configured metric pattern
for bind_pattern in settings.BIND_PATTERNS:
log.listener("binding exchange '%s' to queue '%s' with pattern %s" \
% (exchange, my_queue, bind_pattern))
yield chan.queue_bind(exchange=exchange, queue=my_queue,
routing_key=bind_pattern)
d = chan.queue_bind(exchange=exchange, queue=my_queue,
routing_key=bind_pattern)
wfd = waitForDeferred(d)
yield wfd
d = chan.basic_consume(queue=my_queue, no_ack=True,
consumer_tag=self.consumer_tag)
wfd = waitForDeferred(d)
yield wfd
yield chan.basic_consume(queue=my_queue, no_ack=True,
consumer_tag=self.consumer_tag)
@inlineCallbacks
@deferredGenerator
def receive_loop(self):
queue = yield self.queue(self.consumer_tag)
wfd = waitForDeferred(self.queue(self.consumer_tag))
yield wfd
queue = wfd.getResult()
while True:
msg = yield queue.get()
wfd = waitForDeferred(queue.get())
yield wfd
msg = wfd.getResult()
self.processMessage(msg)
def processMessage(self, message):
......@@ -120,6 +142,8 @@ class AMQPGraphiteProtocol(AMQClient):
else:
value, timestamp = line.split()
datapoint = ( float(timestamp), float(value) )
if datapoint[1] != datapoint[1]: # filter out NaN values
continue
except ValueError:
log.listener("invalid message line: %s" % (line,))