Commit ce2f13f1 authored by Bernd Zeimetz's avatar Bernd Zeimetz

Merge branch 'master' into wheezy-backports

parents 7408326e 2d1e3712
# 2.4 (unreleased master branch)
# 2.6 (unreleased master branch)
# 2.5 (2017-01-09)
### Bugfixes
* [Issue #239](https://github.com/grobian/carbon-c-relay/issues/239)
segfault when date format is incorrect
* [Issue #242](https://github.com/grobian/carbon-c-relay/issues/242)
dispatcher/aggregations broken (relay seems too slow)
# 2.4 (2017-01-03)
### New Features
* **router** `match` rules now support a `validate` clause to do data
filtering, [Issue #228](https://github.com/grobian/carbon-c-relay/issues/228),
[Issue #142](https://github.com/grobian/carbon-c-relay/issues/142),
[Issue #121](https://github.com/grobian/carbon-c-relay/issues/121),
[Pull #127](https://github.com/grobian/carbon-c-relay/issues/127),
[Pull #87](https://github.com/grobian/carbon-c-relay/issues/87).
### Bugfixes
* **server** connection errors are no longer endlessly repeated
* [Issue #240](https://github.com/grobian/carbon-c-relay/issues/240)
'include' directive doesn't care about rewrites.
* [Issue #241](https://github.com/grobian/carbon-c-relay/issues/241)
XXX characters being prepended to metrics when sent via UDP
* [Issue #204](https://github.com/grobian/carbon-c-relay/issues/204)
relay is sending data randomly while kill -HUP happens
# 2.3 (2016-11-07)
......
# Copyright 2013-2016 Fabian Groffen
# Copyright 2013-2017 Fabian Groffen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -15,7 +15,7 @@
CFLAGS ?= -O2 -Wall -Wshadow
GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always || date +%F)
GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always 2>/dev/null || date +%F)
GVCFLAGS += -DGIT_VERSION=\"$(GIT_VERSION)\"
override CFLAGS += $(GVCFLAGS) -pthread
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -180,7 +180,6 @@ aggregator_putmetric(
double val;
long long int epoch;
long long int itime;
int slot;
char newmetric[METRIC_BUFSIZ];
char *newfirstspace = NULL;
size_t len;
......@@ -197,7 +196,7 @@ aggregator_putmetric(
if (__sync_bool_compare_and_swap(&keep_running, 0, 0))
return;
/* get value */
/* get timestamp */
if ((v = strchr(firstspace + 1, ' ')) == NULL) {
/* metric includes \n */
if (mode & MODE_DEBUG)
......@@ -235,14 +234,12 @@ aggregator_putmetric(
((omhash >> AGGR_HT_POW_SIZE) ^ omhash) &
(((unsigned int)1 << AGGR_HT_POW_SIZE) - 1);
#define find_invocation(o) \
invocation = compute->invocations_ht[omhtbucket]; \
for (; invocation != NULL; invocation = invocation->next) \
if (invocation->hash == omhash && \
strcmp(o, invocation->metric) == 0) /* match */ \
break;
pthread_rwlock_wrlock(&compute->invlock);
find_invocation(ometric);
invocation = compute->invocations_ht[omhtbucket];
for (; invocation != NULL; invocation = invocation->next)
if (invocation->hash == omhash &&
strcmp(ometric, invocation->metric) == 0) /* match */
break;
if (invocation == NULL) { /* no match, add */
long long int i;
......@@ -310,8 +307,7 @@ aggregator_putmetric(
continue;
}
slot = itime / s->interval;
if (slot >= s->bucketcnt) {
if (itime >= (s->bucketcnt * s->interval)) {
if (mode & MODE_DEBUG)
logerr("aggregator: dropping metric too far in the "
"future (%lld > %lld): %s from %s", epoch,
......@@ -322,7 +318,7 @@ aggregator_putmetric(
continue;
}
bucket = &invocation->buckets[slot];
bucket = &invocation->buckets[itime / s->interval];
if (bucket->cnt == 0) {
bucket->sum = val;
bucket->max = val;
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
.\" generated with Ronn/v0.7.3
.\" http://github.com/rtomayko/ronn/tree/0.7.3
.
.TH "CARBON\-C\-RELAY" "1" "August 2016" "Graphite" "Graphite data collection and visualisation"
.TH "CARBON\-C\-RELAY" "1" "January 2017" "Graphite" "Graphite data collection and visualisation"
.
.SH "NAME"
\fBcarbon\-c\-relay\fR \- graphite relay, aggregator and rewriter
......@@ -139,6 +139,7 @@ cluster <name>
match
<* | expression \.\.\.>
[validate <expression> else <log | drop>]
send to <cluster \.\.\. | blackhole>
[stop]
;
......@@ -181,7 +182,7 @@ The \fBforward\fR and \fBfile\fR clusters simply send everything they receive to
DNS hostnames are resolved to a single address, according to the preference rules in RFC 3484 \fIhttps://www\.ietf\.org/rfc/rfc3484\.txt\fR\. The \fBany_of\fR cluster has an explicit \fBuseall\fR flag that enables a hostname to resolve to multiple addresses\. Each address returned becomes a cluster destination\.
.
.P
Match rules are the way to direct incoming metrics to one or more clusters\. Match rules are processed top to bottom as they are defined in the file\. It is possible to define multiple matches in the same rule\. Each match rule can send data to one or more clusters\. Since match rules "fall through" unless the \fBstop\fR keyword is added, carefully crafted match expression can be used to target multiple clusters or aggregations\. This ability allows to replicate metrics, as well as send certain metrics to alternative clusters with careful ordering and usage of the \fBstop\fR keyword\. The special cluster \fBblackhole\fR discards any metrics sent to it\. This can be useful for weeding out unwanted metrics in certain cases\. Because throwing metrics away is pointless if other matches would accept the same data, a match with as destination the blackhole cluster, has an implicit \fBstop\fR\.
Match rules are the way to direct incoming metrics to one or more clusters\. Match rules are processed top to bottom as they are defined in the file\. It is possible to define multiple matches in the same rule\. Each match rule can send data to one or more clusters\. Since match rules "fall through" unless the \fBstop\fR keyword is added, carefully crafted match expression can be used to target multiple clusters or aggregations\. This ability allows to replicate metrics, as well as send certain metrics to alternative clusters with careful ordering and usage of the \fBstop\fR keyword\. The special cluster \fBblackhole\fR discards any metrics sent to it\. This can be useful for weeding out unwanted metrics in certain cases\. Because throwing metrics away is pointless if other matches would accept the same data, a match with as destination the blackhole cluster, has an implicit \fBstop\fR\. The \fBvalidation\fR clause adds a check to the data (what comes after the metric) in the form of a regular expression\. When this expression matches, the match rule will execute as if no validation clause was present\. However, if it fails, the match rule is aborted, and no metrics will be sent to destinations, this is the \fBdrop\fR behaviour\. When \fBlog\fR is used, the metric is logged to stderr\. Care should be taken with the latter to avoid log flooding\. When a validate clause is present, destinations need not to be present, this allows for applying a global validation rule\. Note that the cleansing rules are applied before validation is done, thus the data will not have duplicate spaces\.
.
.P
Rewrite rules take a regular input to match incoming metrics, and transform them into the desired new metric name\. In the replacement, backreferences are allowed to match capture groups defined in the input regular expression\. A match of \fBserver\e\.(x|y|z)\e\.\fR allows to use e\.g\. \fBrole\.\e1\.\fR in the substitution\. A few caveats apply to the current implementation of rewrite rules\. First, their location in the config file determines when the rewrite is performed\. The rewrite is done in\-place, as such a match rule before the rewrite would match the original name, a match rule after the rewrite no longer matches the original name\. Care should be taken with the ordering, as multiple rewrite rules in succession can take place, e\.g\. \fBa\fR gets replaced by \fBb\fR and \fBb\fR gets replaced by \fBc\fR in a succeeding rewrite rule\. The second caveat with the current implementation, is that the rewritten metric names are not cleansed, like newly incoming metrics are\. Thus, double dots and potential dangerous characters can appear if the replacement string is crafted to produce them\. It is the responsibility of the writer to make sure the metrics are clean\. If this is an issue for routing, one can consider to have a rewrite\-only instance that forwards all metrics to another instance that will do the routing\. Obviously the second instance will cleanse the metrics as they come in\. The backreference notation allows to lowercase and uppercase the replacement string with the use of the underscore (\fB_\fR) and carret (\fB^\fR) symbols following directly after the backslash\. For example, \fBrole\.\e_1\.\fR as substitution will lowercase the contents of \fB\e1\fR\.
......@@ -387,6 +388,47 @@ match * send to new;
In this example the old cluster would receive the metric that\'s unwanted for the new cluster\. So, the order in which the rules occur does matter for the execution\.
.
.P
Validation can be used to ensure the data for metrics is as expected\. A global validation for just number (no floating point) values could be:
.
.IP "" 4
.
.nf
match *
validate ^[0\-9]+\e [0\-9]+$ else drop
;
.
.fi
.
.IP "" 0
.
.P
(Note the escape with backslash \fB\e\fR of the space, you might be able to use \fB\es\fR or \fB[:space:]\fR instead, this depends on your libc implementation\.)
.
.P
The validation clause can exist on every match rule, so in principle, the following is valid:
.
.IP "" 4
.
.nf
match ^foo
validate ^[0\-9]+\e [0\-9]+$ else drop
send to integer\-cluster
;
match ^foo
validate ^[0\-9\.e+\-]+\e [0\-9\.e+\-]+$ else drop
send to float\-cluster
stop;
.
.fi
.
.IP "" 0
.
.P
Note that the behaviour is different in the previous two examples\. When no \fBsend to\fR clusters are specified, a validation error makes the match behave like the \fBstop\fR keyword is present\. Likewise, when validation passes, processing continues with the next rule\. When destination clusters are present, the \fBmatch\fR respects the \fBstop\fR keyword as normal\. When specified, processing will always stop when specified so\. However, if validation fails, the rule does not send anything to the destination clusters, the metric will be dropped or logged, but never sent\.
.
.P
The relay is capable of rewriting incoming metrics on the fly\. This process is done based on regular expressions with capture groups that allow to substitute parts in a replacement string\. Rewrite rules allow to cleanup metrics from applications, or provide a migration path\. In it\'s simplest form a rewrite rule looks like this:
.
.IP "" 4
......
......@@ -197,6 +197,7 @@ cluster <name>
match
<* | expression ...>
[validate <expression> else <log | drop>]
send to <cluster ... | blackhole>
[stop]
;
......@@ -299,6 +300,16 @@ careful ordering and usage of the `stop` keyword. The special cluster
weeding out unwanted metrics in certain cases. Because throwing metrics
away is pointless if other matches would accept the same data, a match
with as destination the blackhole cluster, has an implicit `stop`.
The `validation` clause adds a check to the data (what comes after the
metric) in the form of a regular expression. When this expression
matches, the match rule will execute as if no validation clause was
present. However, if it fails, the match rule is aborted, and no
metrics will be sent to destinations, this is the `drop` behaviour.
When `log` is used, the metric is logged to stderr. Care should be
taken with the latter to avoid log flooding. When a validate clause is
present, destinations need not to be present, this allows for applying a
global validation rule. Note that the cleansing rules are applied
before validation is done, thus the data will not have duplicate spaces.
Rewrite rules take a regular input to match incoming metrics, and
transform them into the desired new metric name. In the replacement,
......@@ -538,6 +549,43 @@ In this example the old cluster would receive the metric that's unwanted
for the new cluster. So, the order in which the rules occur does
matter for the execution.
Validation can be used to ensure the data for metrics is as expected. A
global validation for just number (no floating point) values could be:
```
match *
validate ^[0-9]+\ [0-9]+$ else drop
;
```
(Note the escape with backslash `\` of the space, you might be able to
use `\s` or `[:space:]` instead, this depends on your libc
implementation.)
The validation clause can exist on every match rule, so in principle,
the following is valid:
```
match ^foo
validate ^[0-9]+\ [0-9]+$ else drop
send to integer-cluster
;
match ^foo
validate ^[0-9.e+-]+\ [0-9.e+-]+$ else drop
send to float-cluster
stop;
```
Note that the behaviour is different in the previous two examples. When
no `send to` clusters are specified, a validation error makes the match
behave like the `stop` keyword is present. Likewise, when validation
passes, processing continues with the next rule.
When destination clusters are present, the `match` respects the `stop`
keyword as normal. When specified, processing will always stop when
specified so. However, if validation fails, the rule does not send
anything to the destination clusters, the metric will be dropped or
logged, but never sent.
The relay is capable of rewriting incoming metrics on the fly. This
process is done based on regular expressions with capture groups that
allow to substitute parts in a replacement string. Rewrite rules allow
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
carbon-c-relay (2.3-1~bpo7+1) wheezy-backports-sloppy; urgency=medium
carbon-c-relay (2.5-1~bpo7+1) wheezy-backports-sloppy; urgency=medium
* Rebuild for wheezy-backports-sloppy.
-- Bernd Zeimetz <bzed@debian.org> Fri, 02 Dec 2016 11:28:02 +0100
-- Bernd Zeimetz <bzed@debian.org> Wed, 08 Mar 2017 11:27:36 +0100
carbon-c-relay (2.5-1) unstable; urgency=medium
* [770fae3] Merge tag 'upstream/2.5'
Upstream version 2.5
* [c6a37a5] Refreshing patches
-- Bernd Zeimetz <bzed@debian.org> Mon, 16 Jan 2017 22:18:31 +0100
carbon-c-relay (2.3-1) unstable; urgency=medium
......
......@@ -4,7 +4,7 @@
CFLAGS ?= -O2 -Wall -Wshadow
-GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always || date +%F)
-GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always 2>/dev/null || date +%F)
+GIT_VERSION := $(shell dpkg-parsechangelog | awk '/^Version:/ {print $$2}')
GVCFLAGS += -DGIT_VERSION=\"$(GIT_VERSION)\"
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -56,6 +56,7 @@ typedef struct _connection {
unsigned int maxsenddelay;
char hadwork:1;
char isaggr:1;
char isudp:1;
} connection;
struct _dispatcher {
......@@ -270,6 +271,7 @@ dispatch_addconnection(int sock)
connections[c].needmore = 0;
connections[c].noexpire = 0;
connections[c].isaggr = 0;
connections[c].isudp = 0;
connections[c].destlen = 0;
gettimeofday(&connections[c].lastwork, NULL);
connections[c].hadwork = 1; /* force first iteration before stalling */
......@@ -316,6 +318,7 @@ dispatch_addlistener_udp(int sock)
return 1;
connections[conn].noexpire = 1;
connections[conn].isudp = 1;
acceptedconnections--;
return 0;
......@@ -518,19 +521,19 @@ dispatch_connection(connection *conn, dispatcher *self, struct timeval start)
return 0;
}
}
if (len == -1 || len == 0) { /* error + EOF */
if (len == -1 || len == 0 || conn->isudp) { /* error + EOF */
/* we also disconnect the client in this case if our reading
* buffer is full, but we still need more (read returns 0 if the
* size argument is 0) -> this is good, because we can't do much
* with such client */
if (conn->noexpire) {
/* reset buffer only (UDP) and move on */
/* reset buffer only (UDP/aggregations) and move on */
conn->needmore = 1;
conn->buflen = 0;
__sync_bool_compare_and_swap(&(conn->takenby), self->id, 0);
return 0;
return len > 0;
} else {
__sync_add_and_fetch(&closedconnections, 1);
close(conn->sock);
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
cluster foo file ip /dev/stdout;
match foo.bar
validate ^[0-9.e+-]+\ [0-9.e+-]+$ else drop
send to foo
;
match foo.bar
validate ^[0-9]+\ [0-9]+$ else drop
send to foo
stop
;
match *
validate ^[0-9]+\ [0-9]+$ else drop
;
match *
validate ^unknown+\ [0-9]+$ else log
stop
;
cluster local_carbon
forward
127.0.0.1:2013
;
match *
validate ^[-+]?[0-9]*.?[0-9]+([eE][-+]?[0-9]+)?\ [0-9]{10}$ else log
;
match *
validate ^[-+]?[0-9]*.?[0-9]+([eE][-+]?[0-9]+)?\ [0-9]{10}$ else log
send to local_carbon
stop
;
cluster zone1
carbon_ch replication 1
127.0.0.1:2101
;
cluster zone2
carbon_ch replication 1
127.0.0.1:2102
;
cluster zone3
carbon_ch replication 1
127.0.0.1:2103
;
match *
validate ^[-+]?[0-9]*.?[0-9]+([eE][-+]?[0-9]+)?\ [0-9.]+$ else drop
send to zone1
;
match *
validate ^[-+]?[0-9]*.?[0-9]+([eE][-+]?[0-9]+)?\s[0-9.]+$ else drop
send to zone2
;
match *
validate ^[-+]?[0-9]*.?[0-9]+([eE][-+]?[0-9]+)?[:space:][0-9.]+$ else drop
send to zone3
stop
;
cluster default
forward
127.0.0.1:2013
;
include issues/issue240-rewrites.conf
;
match *
send to default
;
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -415,7 +415,7 @@ main(int argc, char * const argv[])
case 'L':
maxstalls = atoi(optarg);
if (maxstalls < 0 || maxstalls >= (1 << SERVER_STALL_BITS)) {
fprintf(stderr, "error: maxium stalls needs to be a number "
fprintf(stderr, "error: maximum stalls needs to be a number "
"between 0 and %d\n", (1 << SERVER_STALL_BITS) - 1);
do_usage(argv[0], 1);
}
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -18,7 +18,7 @@
#ifndef HAVE_RELAY_H
#define HAVE_RELAY_H 1
#define VERSION "2.3"
#define VERSION "2.5"
#define METRIC_BUFSIZ 8192
......
This diff is collapsed.
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......@@ -94,8 +94,6 @@ server_queuereader(void *d)
const char *p;
*metric = NULL;
__sync_and_and_fetch(&(self->metrics), 0);
__sync_and_and_fetch(&(self->ticks), 0);
#define FAIL_WAIT_TIME 6 /* 6 * 250ms = 1.5s */
#define DISCONNECT_WAIT_TIME 12 /* 12 * 250ms = 3s */
......@@ -224,6 +222,9 @@ server_queuereader(void *d)
if (__sync_bool_compare_and_swap(&(self->keep_running), 0, 0))
break;
usleep((200 + (rand() % 100)) * 1000); /* 200ms - 300ms */
/* avoid overflowing */
if (__sync_add_and_fetch(&(self->failure), 0) > FAIL_WAIT_TIME)
__sync_sub_and_fetch(&(self->failure), 1);
}
/* at this point we've got work to do, if we're instructed to
......@@ -429,7 +430,7 @@ server_queuereader(void *d)
__sync_fetch_and_add(&(self->failure), 1) == 0)
logerr("failed to write() to %s:%u: %s\n",
self->ip, self->port,
(slen < 0 ? strerror(errno) : "uncomplete write"));
(slen < 0 ? strerror(errno) : "incomplete write"));
close(self->fd);
self->fd = -1;
/* put back stuff we couldn't process */
......@@ -710,6 +711,16 @@ server_swap_queue(server *l, server *r)
t = l->queue;
l->queue = r->queue;
r->queue = t;
/* swap associated statistics as well */
l->metrics = r->metrics;
l->dropped = r->dropped;
l->stalls = r->stalls;
l->ticks = r->ticks;
l->prevmetrics = r->prevmetrics;
l->prevdropped = r->prevdropped;
l->prevstalls = r->prevstalls;
l->prevticks = r->prevticks;
}
/**
......
/*
* Copyright 2013-2016 Fabian Groffen
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment