Commit 6b23f226 authored by Bernd Zeimetz's avatar Bernd Zeimetz

New upstream version 3.2

parent 586f43ee
*.o
/.deps/
/Makefile
/autom4te.cache/
/config.h
/config.h.in~
/config.log
/config.status
/stamp-h1
/relay
/carbon-c-relay-*.tar.gz
script: make
language: c
os:
- linux
- osx
compiler:
- gcc
- clang
matrix:
exclude:
include:
- compiler: gcc
os: linux
- compiler: clang
os: linux
- compiler: coverity
os: linux
env:
- COVERITY_SCAN_PROJECT_NAME="$TRAVIS_REPO_SLUG"
- COVERITY_SCAN_BRANCH_PATTERN="master"
- COVERITY_SCAN_NOTIFICATION_EMAIL="grobian@gentoo.org"
- COVERITY_SCAN_BUILD_COMMAND="make"
- compiler: clang
os: osx
# gcc is actually clang on OSX, so pointless to test twice
- os: osx
compiler: gcc
script:
- |
if [ $TRAVIS_OS_NAME = osx ]; then
brew update > /dev/null
brew install lz4 openssl
fi
# workaround git not retaining mtimes and bison/flex not being uptodate
- touch conffile.yy.c conffile.tab.c conffile.tab.h
- touch configure.ac Makefile.am aclocal.m4 configure Makefile.in config.h.in
- |
if [[ ${CC} == coverity ]] ; then
# ensure we end up with an existing compiler
export CC=gcc
./configure || { cat config.log ; exit 1 ; }
curl -s 'https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh' | bash
else
# everything disabled, this MUST work
./configure --without-gzip --without-lz4 --without-ssl \
|| { cat config.log ; exit 1 ; }
make CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" clean check || exit 1
# compile some enabled/disabled variants compile only
if ./configure --with-gzip --without-lz4 --without-ssl ; then
make CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" clean relay || exit 1
fi
if ./configure --without-gzip --with-lz4 --without-ssl ; then
make CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" clean relay || exit 1
fi
if ./configure --with-gzip --with-lz4 --without-ssl ; then
make CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" clean relay || exit 1
fi
if ./configure --with-gzip --with-lz4 --with-ssl ; then
make CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" clean relay || exit 1
fi
# final test with everything enabled that is detected
./configure || { cat config.log ; exit 1 ; }
make CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" clean check || exit 1
fi
env:
global:
# The next declaration is the encrypted COVERITY_SCAN_TOKEN, created
# via the "travis encrypt" command using the project repo's public key
- secure: "H844xDJC5CUXhUyB1SFsGiiz/Q4viVSALkj/KnWAuagrbrzIXJoK4tRWpPxCzUar5QsdEua1WIXTBezPA6V97w2cis2OGUXtgsMd1fWnwlG0RwPRj2+LAVxJ+i5z1Asx8hkYOiZIv8y0VkBGldjOrD0l0pBcxB+CDZC/zsGSRtnMLOs+smfEjOF6JPGw8Ye0XJ+YKrfbHOXhfS7XTMwUnjgAV7lTH/aAlKuedc3mYsuC3WHJOMptYLXaiPdbPKoIXfLdmszP91SkThr4f0elyQskugzGM3D7M1tIFhfmordSD+DfzLvvdSu32hHUNQjHvAm7Pz8QmdLC2IjjE8y8MkIkEy2+wR/5AjacnKR8OgFzXgt+yIH+CBc6qbTi40dZ7Go33Q8uNyKJGrA1CyL195NS8q0reMbuS7WIxI0rNOy7cdw8MO0sNjeofcb1RW90U5u5qciRQpbwhFwMFMCuD0LL41mjfN+IHGNeUwp148Qibbvyu+d7y4Y1F7/CsdMpZVBH9Ae7CHXetN8biT81fJp1rn0uEGQqoPuxvKeePsL40MXu6wvlrSf5wLKV3aqp5lzbjoJsPrOwV5iVwMHrvdhoAFlLc6Y1aJkCTPPt7yjLHgLBisqRXLTZ9lcD3AG8ubTLr03DsseUUM3/tWvhWfVRfxzkuOHsDRLMfdY2Q20="
- CFLAGS="-O3 -Wall -Werror -Wshadow -pipe"
addons:
coverity_scan:
project:
name: "grobian/carbon-c-relay"
description: "Build submitted via Travis CI"
notification_email: grobian@gentoo.org
build_command_prepend: "make clean"
build_command: "make -j 4"
branch_pattern: master
# 2.6 (unreleased master branch)
# 3.3 (unreleased master branch)
# 3.2 (21-10-2017)
### Notes
* The deprecated `-S` and `-m` options were removed. Use the
`statistics` configuration file construct instead.
* The `-i` option was removed. Use tue `listen` configuration file
construct instead.
### New Features
* **relay** the flag `-tt` can now be used to test the configuration and
exit immediately. In this mode, any standard output is suppressed as
well.
[Issue #247](https://github.com/grobian/carbon-c-relay/issues/247)
* **relay** the `listen` construct was added to precisely control listen
ports and protocols.
* **server** unresolvable hosts are now treated as failed servers, this
means the relay will only warn about unresolvable hosts, but not error
out. If a host becomes unresolvable this server is transitioned to
failed state until it is resolvable again.
[Issue #293](https://github.com/grobian/carbon-c-relay/issues/293)
* **relay** compression and encryption support was added to listeners
and servers.
[Issue #212](https://github.com/grobian/carbon-c-relay/issues/212)
[Issue #75](https://github.com/grobian/carbon-c-relay/issues/75)
### Bugfixes
* [Issue #265](https://github.com/grobian/carbon-c-relay/issues/265)
exit code is sucess on error when daemonised
* [Issue #266](https://github.com/grobian/carbon-c-relay/issues/266)
incorrect hash when using aggregate function
* [Issue #267](https://github.com/grobian/carbon-c-relay/issues/267)
segmentation fault when trying to use a file cluster
* [PR #273](https://github.com/grobian/carbon-c-relay/pull/273)
regex matching slow on glibc due to lock contention
* [Issue #287](https://github.com/grobian/carbon-c-relay/issues/287)
unexpected warning about rejected socket bufsize
* [PR #282](https://github.com/grobian/carbon-c-relay/pull/282)
route using should not change the metric key sent
* [PR #274](https://github.com/grobian/carbon-c-relay/pull/274)
aggregator expiry deadlocks under pressure
* [buckytools Issue #17](https://github.com/jjneely/buckytools/issues/17)
consistent-hash implementation returns a different distribution with
the binary search approach
# 3.1 (29-04-2017)
### New Features
* **router** a new `route using` clause was added to `match` rules to
set the string to be used as key for consistent-hash based clusters.
[Pull Request #235](https://github.com/grobian/carbon-c-relay/pull/235)
* **consistent-hash** node traversal is now done through a binary search
instead of forward pointer chasing.
### Bugfixes
* [Issue #261](https://github.com/grobian/carbon-c-relay/issues/261)
unable to use numeric instance name
* [Issue #262](https://github.com/grobian/carbon-c-relay/issues/262)
DNS round-robin on any\_of cluster doesn't rotate and causes lots of
hung connections
* [Issue #263](https://github.com/grobian/carbon-c-relay/issues/263)
parser fails on `%` in regular expressions, and exits with a crash
* [Issue #264](https://github.com/grobian/carbon-c-relay/issues/264)
carbon\_ch hash algorithm change between 2.6 and 3.0
# 3.0 (2017-04-07)
### Notes
* Parsing of the configuration file has changed implementation in this
version. Should you find problems with configurations that worked
fine before, please report the problem.
* The use of `send statistics to` is now deprecated, use the new
`statistics send to` instead.
* The use of statistics controlling flags `-S` sending interval and `-m`
summary mode, are now deprecated because they can be set in the
configuration.
### New Features
* **collector** statistics now can be controlled via the configuration
file using the new `statistics` clause. This overrides some command
line flags, and adds some new posibilities.
* **router** the rewrite rule got a new replacement mode: `\.` This
mode replaces all dots with underscores in the matched group. A
combination with upper/lowercasing is possible, e.g. `\_.`.
* **collector** one can now set the internal statistics prefix
[Issue #25](https://github.com/grobian/carbon-c-relay/issues/25)
[Issue #113](https://github.com/grobian/carbon-c-relay/issues/113)
[Issue #249](https://github.com/grobian/carbon-c-relay/issues/249).
### Bugfixes
* [Issue #249](https://github.com/grobian/carbon-c-relay/issues/249)
`send statistics` changes metric prefix from dots into underscores
# 2.6 (2017-01-26)
### New Features
* **router** `useall` is now also allowed for `forward` and `failover`
clusters
* **server** hostnames are now re-resolved at every connect
[Issue #115](https://github.com/grobian/carbon-c-relay/issues/115)
* **server** when connecting to a hostname, now /all/ resolved addresses
are tried in order before failing
* **relay** A new `-O` flag allows to set the minimum rules threshold
before trying to optimise the rules.
### Bugfixes
* [Issue #246](https://github.com/grobian/carbon-c-relay/issues/246)
using `stop` in a `match` rule does not stop metrics from being
processed afterwards
* [Issue #206](https://github.com/grobian/carbon-c-relay/issues/206)
some metrics are randomly prefixed with garbage characters
# 2.5 (2017-01-09)
......
FROM alpine:3.6
MAINTAINER Fabian Groffen
RUN mkdir -p /opt/carbon-c-relay-build
RUN mkdir /etc/carbon-c-relay
COPY . /opt/carbon-c-relay-build
RUN \
apk --no-cache update && \
apk --no-cache upgrade && \
apk --no-cache add git bc build-base curl && \
cd /opt/carbon-c-relay-build && \
./configure; make && \
cp relay /usr/bin/carbon-c-relay && \
apk del --purge git bc build-base ca-certificates curl && \
rm -rf /opt/* /tmp/* /var/cache/apk/* /opt/carbon-c-relay-build
EXPOSE 2003
ENTRYPOINT ["carbon-c-relay", "-f", "/etc/carbon-c-relay/carbon-c-relay.conf"]
......@@ -12,38 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
AUTOMAKE_OPTIONS = foreign
CFLAGS ?= -O2 -Wall -Wshadow
CFLAGS ?= -O2 -Wall -Wshadow -pipe
GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always 2>/dev/null || date +%F)
GVCFLAGS += -DGIT_VERSION=\"$(GIT_VERSION)\"
GVCFLAGS = -DGIT_VERSION=\"$(GIT_VERSION)\"
override CFLAGS += $(GVCFLAGS) -pthread
SOCKET_LIBS =
ifeq ($(shell uname), SunOS)
SOCKET_LIBS += -lsocket -lnsl
endif
conffile.tab.c conffile.tab.h: conffile.y
bison -d $^
# should be accepted sort of anywhere
MATH_LIBS = -lm
override LIBS += $(SOCKET_LIBS) $(MATH_LIBS) -pthread
OBJS = \
relay.o \
md5.o \
consistent-hash.o \
receptor.o \
dispatcher.o \
router.o \
queue.o \
server.o \
collector.o \
aggregator.o
relay: $(OBJS)
$(CC) -o $@ $(LDFLAGS) $^ $(LIBS)
conffile.yy.c: conffile.l conffile.tab.h
flex -o $@ $<
man:
sed -e '/travis-ci.org\/grobian\/carbon-c-relay.svg/d' carbon-c-relay.md | \
......@@ -53,12 +35,82 @@ man:
--roff \
> carbon-c-relay.1
VERSION = $(shell sed -n '/VERSION/s/^.*"\([0-9.]\+\)".*$$/\1/p' relay.h)
dist:
git archive \
--format=tar.gz \
--prefix=carbon-c-relay-$(VERSION)/ v$(VERSION) \
> carbon-c-relay-$(VERSION).tar.gz
bin_PROGRAMS = relay
relay_LINK = $(CCLD) $(AM_LDFLAGS) $(LDFLAGS) -o $@
relay_LDADD = $(LIBGZIP) $(LIBLZ4) $(LIBSSL)
relay_SOURCES = \
relay.c \
md5.c \
consistent-hash.c \
receptor.c \
dispatcher.c \
conffile.tab.c \
conffile.yy.c \
allocator.c \
router.c \
queue.c \
server.c \
collector.c \
aggregator.c
noinst_HEADERS = \
relay.h \
md5.h \
fnv1a.h \
consistent-hash.h \
conffile.h \
conffile.tab.h \
receptor.h \
dispatcher.h \
allocator.h \
router.h \
queue.h \
server.h \
collector.h \
aggregator.h
man_MANS = carbon-c-relay.1
EXTRA_DIST = \
issues \
test \
ChangeLog.md \
LICENSE.md \
carbon-c-relay.md \
carbon-c-relay.1 \
README.md \
conffile.tab.c conffile.tab.h conffile.y \
conffile.yy.c conffile.l \
$(NULL)
CRTESTS = \
issue10 \
issue27 \
issue117 \
issue156 \
issue157 \
issue163 \
issue165 \
issue180 \
issue184 \
issue202 \
issue213 \
issue218 \
issue228 \
issue235 \
issue236 \
issue246 \
issue252 \
issue253 \
issue263 \
issue267 \
issue288 \
issue293 \
issue310 \
$(NULL)
clean:
rm -f *.o relay
test: check
test-%: relay
@( cd test && ./run-test.sh $* )
approve-%: relay
@( cd test && ./run-test.sh -a $* )
check: relay
@( cd test && ./run-test.sh $(CRTESTS) )
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -43,6 +43,8 @@ typedef struct _aggregator {
unsigned int hash; /* to speed up matching */
unsigned short expire; /* expire + splay */
struct _aggr_bucket {
pthread_mutex_t bcklock;
enum { A_OPEN, A_EXPIRE } state;
long long int start;
size_t cnt;
double sum;
......@@ -64,7 +66,7 @@ typedef struct _aggregator {
} aggregator;
aggregator *aggregator_new(unsigned int interval, unsigned int expire, enum _aggr_timestamp tswhen);
char aggregator_add_compute(aggregator *s, const char *metric, const char *type);
char aggregator_add_compute(aggregator *s, const char *metric, enum _aggr_compute_type type, unsigned char pctl);
void aggregator_set_stub(aggregator *s, const char *stubname);
void aggregator_putmetric(aggregator *s, const char *metric, const char *firstspace, size_t nmatch, regmatch_t *pmatch);
int aggregator_start(aggregator *aggrs);
......
/*
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "allocator.h"
struct _cr_allocator {
void *memory_region;
void *nextp;
size_t sz;
struct _cr_allocator *next;
};
/**
* Free the resources associated to this allocator.
*/
void
ra_free(allocator *ra)
{
allocator *ra_next;
for ( ; ra != NULL; ra = ra_next) {
free(ra->memory_region);
ra_next = ra->next;
free(ra);
ra = NULL;
}
}
#define ra_alloc(RA, SZ) { \
assert(SZ >= 0); \
nsz = 256 * 1024; \
if (SZ > nsz) \
nsz = ((SZ / 1024) + 1) * 1024; \
RA = malloc(sizeof(allocator)); \
if (RA == NULL) \
return NULL; \
RA->memory_region = malloc(sizeof(char) * nsz); \
if (RA->memory_region == NULL) { \
free(RA); \
RA = NULL; \
return NULL; \
} \
RA->nextp = RA->memory_region; \
RA->sz = nsz; \
RA->next = NULL; \
}
/**
* Allocate a new allocator.
*/
allocator *
ra_new(void)
{
allocator *ret;
size_t nsz;
ra_alloc(ret, 0);
return ret;
}
/**
* malloc() in one of the regions for this allocator. If insufficient
* memory is available in the region, a new one is allocated. If
* that fails, NULL is returned, else a pointer that can be written to
* up to sz bytes. The returned region is aligned.
*/
void *
ra_malloc(allocator *ra, size_t sz)
{
void *retp = NULL;
size_t nsz; /* for ra_alloc macro */
for (; ra != NULL; ra = ra->next) {
if (ra->sz - (ra->nextp - ra->memory_region) >= sz) {
retp = ra->nextp;
/* align to arch-width boundaries */
nsz = sz % sizeof(size_t);
if (nsz != 0)
sz += sizeof(size_t) - nsz;
ra->nextp += sz;
return retp;
}
if (ra->next == NULL)
ra_alloc(ra->next, sz);
}
/* this should be unreachable code */
return NULL;
}
/**
* strdup using ra_malloc, e.g. get memory from the region associated
* to the given allocated.
*/
char *
ra_strdup(allocator *ra, const char *s)
{
size_t sz = strlen(s) + 1;
char *m = ra_malloc(ra, sz);
if (m == NULL)
return m;
memcpy(m, s, sz);
return m;
}
/*
* Copyright 2013-2017 Fabian Groffen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef _ALLOCATOR_H
#define _ALLOCATOR_H 1
#include <stdlib.h>
typedef struct _cr_allocator allocator;
void ra_free(allocator *ra);
allocator *ra_new(void);
void *ra_malloc(allocator *ra, size_t sz);
char *ra_strdup(allocator *ra, const char *s);
#endif
This diff is collapsed.
This diff is collapsed.
......@@ -32,8 +32,6 @@ static dispatcher **dispatchers;
static char debug = 0;
static pthread_t collectorid;
static char keep_running = 1;
int collector_interval = 60;
static char cluster_refresh_pending = 0;
static router *pending_router = NULL;
/**
......@@ -61,6 +59,8 @@ collector_runner(void *s)
time_t nextcycle;
char destbuf[1024]; /* sort of POSIX_MAX_PATH */
char *p;
int collector_interval = 60;
router *prtr;
size_t numaggregators = 0;
aggregator *aggrs = NULL;
server *submission = (server *)s;
......@@ -68,44 +68,17 @@ collector_runner(void *s)
char metric[METRIC_BUFSIZ];
char *m = NULL;
size_t sizem = 0;
size_t (*s_ticks)(server *);
size_t (*s_metrics)(server *);
size_t (*s_stalls)(server *);
size_t (*s_dropped)(server *);
size_t (*d_ticks)(dispatcher *);
size_t (*d_metrics)(dispatcher *);
size_t (*d_blackholes)(dispatcher *);
size_t (*d_sleeps)(dispatcher *);
size_t (*a_received)(aggregator *);
size_t (*a_sent)(aggregator *);
size_t (*a_dropped)(aggregator *);
/* setup functions to target what the user wants */
if (debug & 2) {
s_ticks = server_get_ticks_sub;
s_metrics = server_get_metrics_sub;
s_stalls = server_get_stalls_sub;
s_dropped = server_get_dropped_sub;
d_ticks = dispatch_get_ticks_sub;
d_metrics = dispatch_get_metrics_sub;
d_blackholes = dispatch_get_blackholes_sub;
d_sleeps = dispatch_get_sleeps_sub;
a_received = aggregator_get_received_sub;
a_sent = aggregator_get_sent_sub;
a_dropped = aggregator_get_dropped_sub;
} else {
s_ticks = server_get_ticks;
s_metrics = server_get_metrics;
s_stalls = server_get_stalls;
s_dropped = server_get_dropped;
d_ticks = dispatch_get_ticks;
d_metrics = dispatch_get_metrics;
d_blackholes = dispatch_get_blackholes;
d_sleeps = dispatch_get_sleeps;
a_received = aggregator_get_received;
a_sent = aggregator_get_sent;
a_dropped = aggregator_get_dropped;
}
size_t (*s_ticks)(server *) = NULL;
size_t (*s_metrics)(server *) = NULL;
size_t (*s_stalls)(server *) = NULL;
size_t (*s_dropped)(server *) = NULL;
size_t (*d_ticks)(dispatcher *) = NULL;
size_t (*d_metrics)(dispatcher *) = NULL;
size_t (*d_blackholes)(dispatcher *) = NULL;
size_t (*d_sleeps)(dispatcher *) = NULL;
size_t (*a_received)(aggregator *) = NULL;
size_t (*a_sent)(aggregator *) = NULL;
size_t (*a_dropped)(aggregator *) = NULL;
#define send(metric) \
if (debug & 1) \
......@@ -115,26 +88,57 @@ collector_runner(void *s)
nextcycle = time(NULL) + collector_interval;
while (__sync_bool_compare_and_swap(&keep_running, 1, 1)) {
if (cluster_refresh_pending) {
char *stub = router_getcollectorstub(pending_router);
server **newservers = router_getservers(pending_router);
if ((prtr = __sync_add_and_fetch(&pending_router, 0)) != NULL) {
char *stub;
char *prefix;
server **newservers;
if (srvs != NULL)
free(srvs);
srvs = newservers;
aggrs = router_getaggregators(pending_router);
stub = router_getcollectorstub(prtr);
prefix = router_getcollectorprefix(prtr);
srvs = newservers = router_getservers(prtr);
aggrs = router_getaggregators(prtr);
numaggregators = aggregator_numaggregators(aggrs);
collector_interval = router_getcollectorinterval(prtr);
nextcycle = time(NULL) + collector_interval;
/* setup functions to target what the user wants */
if (router_getcollectormode(prtr) == SUB) {
s_ticks = server_get_ticks_sub;
s_metrics = server_get_metrics_sub;
s_stalls = server_get_stalls_sub;
s_dropped = server_get_dropped_sub;
d_ticks = dispatch_get_ticks_sub;
d_metrics = dispatch_get_metrics_sub;
d_blackholes = dispatch_get_blackholes_sub;
d_sleeps = dispatch_get_sleeps_sub;
a_received = aggregator_get_received_sub;
a_sent = aggregator_get_sent_sub;
a_dropped = aggregator_get_dropped_sub;
} else {
s_ticks = server_get_ticks;
s_metrics = server_get_metrics;
s_stalls = server_get_stalls;
s_dropped = server_get_dropped;
d_ticks = dispatch_get_ticks;
d_metrics = dispatch_get_metrics;
d_blackholes = dispatch_get_blackholes;
d_sleeps = dispatch_get_sleeps;
a_received = aggregator_get_received;
a_sent = aggregator_get_sent;
a_dropped = aggregator_get_dropped;
}
/* prepare hostname for graphite metrics */
snprintf(metric, sizeof(metric), "%scarbon.relays.%s",
stub == NULL ? "" : stub, relay_hostname);
for (m = metric + strlen("carbon.relays."); *m != '\0'; m++)
if (*m == '.')
*m = '_';
/* prepare prefix for graphite metrics */
i = snprintf(metric, sizeof(metric), "%s%s",
stub == NULL ? "" : stub, prefix);
m = metric + i;
*m++ = '.';
*m = '\0';
sizem = sizeof(metric) - (m - metric);
cluster_refresh_pending = 0;
(void)__sync_and_and_fetch(&pending_router, NULL);
}
assert(srvs != NULL);
sleep(1);
......@@ -301,18 +305,21 @@ collector_writer(void *unused)
size_t lastconn = 0;
size_t lastdisc = 0;
size_t numaggregators = 0;
router *prtr;
server **srvs = NULL;