Commit 4d4713d0 authored by Sébastien Delafond's avatar Sébastien Delafond

New upstream version 0.2.0b2

parent 0a63478f
root = true
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.py]
indent_style = space
indent_size = 4
[Makefile]
indent_style = tab
[{*.json,*.yml,*.yaml}]
indent_style = space
indent_size = 2
......@@ -12,6 +12,7 @@ venv/
env/
docs/build/
build/
.pytest_cache/
my_rules
*.swp
*~
......@@ -17,3 +17,9 @@ repos:
sha: v0.3.5
hooks:
- id: reorder-python-imports
- repo: git://github.com/Yelp/detect-secrets
sha: 0.9.1
hooks:
- id: detect-secrets
args: ['--baseline', '.secrets.baseline']
exclude: .*tests/.*|.*yelp/testing/.*|\.pre-commit-config\.yaml
{
"exclude_regex": ".*tests/.*|.*yelp/testing/.*|\\.pre-commit-config\\.yaml",
"generated_at": "2018-07-06T22:54:22Z",
"plugins_used": [
{
"base64_limit": 4.5,
"name": "Base64HighEntropyString"
},
{
"hex_limit": 3,
"name": "HexHighEntropyString"
},
{
"name": "PrivateKeyDetector"
}
],
"results": {
".travis.yml": [
{
"hashed_secret": "4f7a1ea04dafcbfee994ee1d08857b8aaedf8065",
"line_number": 14,
"type": "Base64 High Entropy String"
}
]
},
"version": "0.9.1"
}
......@@ -6,7 +6,30 @@ env:
- TOXENV=py27
install:
- pip install tox
script: make test
- >
if [[ -n "${ES_VERSION}" ]] ; then
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1
./elasticsearch-${ES_VERSION}/bin/elasticsearch &
fi
script:
- >
if [[ -n "${ES_VERSION}" ]] ; then
wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200
make test-elasticsearch
else
make test
fi
jobs:
include:
- stage: 'Elasticsearch test'
env: TOXENV=py27 ES_VERSION=7.0.0-linux-x86_64
- env: TOXENV=py27 ES_VERSION=6.6.2
- env: TOXENV=py27 ES_VERSION=6.3.2
- env: TOXENV=py27 ES_VERSION=6.2.4
- env: TOXENV=py27 ES_VERSION=6.0.1
- env: TOXENV=py27 ES_VERSION=5.6.16
deploy:
provider: pypi
user: yelplabs
......
FROM ubuntu:latest
RUN apt-get update && apt-get upgrade -y
RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox
RUN easy_install pip
RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip
WORKDIR /home/elastalert
......
......@@ -16,6 +16,9 @@ install-hooks:
test:
tox
test-elasticsearch:
tox -- --runelasticsearch
test-docker:
docker-compose --project-name elastalert build tox
docker-compose --project-name elastalert run tox
......
......@@ -47,12 +47,15 @@ Currently, we have built-in support for the following alert types:
- MS Teams
- Slack
- Telegram
- GoogleChat
- AWS SNS
- VictorOps
- PagerDuty
- PagerTree
- Exotel
- Twilio
- Gitter
- Line Notify
Additional rule types and alerts can be easily imported or written.
......@@ -88,11 +91,11 @@ Eg: ``--rule this_rule.yaml``
## Third Party Tools And Extras
### Kibana plugin
![img](https://www.bitsensor.io/assets/img/screenshots/template.gif)
Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin)
![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif)
Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin).
### Docker
A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`.
A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`.
```bash
git clone https://github.com/bitsensor/elastalert.git; cd elastalert
......
# Change Log
# v0.1.38
### Added
- Added PagerTree alerter
- Added Line alerter
- Added more customizable logging
- Added new logic in test-rule to detemine the default timeframe
### Fixed
- Fixed an issue causing buffer_time to sometimes be ignored
# v0.1.37
### Added
- Added more options for Opsgenie alerter
- Added more pagerduty options
- Added ability to add metadata to elastalert logs
### Fixed
- Fixed some documentation to be more clear
- Stop requiring doc_type for metric aggregations
- No longer puts quotes around regex terms in blacklists or whitelists
# v0.1.36
### Added
- Added a prefix "metric_" to the key used for metric aggregations to avoid possible conflicts
- Added option to skip Alerta certificate validation
### Fixed
- Fixed a typo in the documentation for spike rule
# v0.1.35
### Fixed
- Fixed an issue preventing new term rule from working with terms query
# v0.1.34
### Added
- Added prefix/suffix support for summary table
- Added support for ignoring SSL validation in Slack
- More visible exceptions during query parse failures
### Fixed
- Fixed top_count_keys when using compound query_key
- Fixed num_hits sometimes being reported too low
- Fixed an issue with setting ES_USERNAME via env
- Fixed an issue when using test script with custom timestamps
- Fixed a unicode error when using Telegram
- Fixed an issue with jsonschema version conflict
- Fixed an issue with nested timestamps in cardinality type
# v0.1.33
### Added
- Added ability to pipe alert text to a command
- Add --start and --end support for elastalert-test-rule
- Added ability to turn blacklist/whitelist files into queries for better performance
- Allow setting of OpsGenie priority
- Add ability to query the adjacent index if timestamp_field not used for index timestamping
- Add support for pagerduty v2
- Add option to turn off .raw/.keyword field postfixing in new term rule
- Added --use-downloaded feature for elastalert-test-rule
### Fixed
- Fixed a bug that caused num_hits in matches to sometimes be erroneously small
- Fixed an issue with HTTP Post alerter that could cause it to hang indefinitely
- Fixed some issues with string formatting for various alerters
- Fixed a couple of incorrect parts of the documentation
# v0.1.32
### Added
- Add support for setting ES url prefix via environment var
- Add support for using native Slack fields in alerts
### Fixed
- Fixed a bug that would could scrolling queries to sometimes terminate early
# v0.1.31
### Added
- Added ability to add start date to new term rule
### Fixed
- Fixed a bug in create_index which would try to delete a nonexistent index
- Apply filters to new term rule all terms query
- Support Elasticsearch 6 for new term rule
- Fixed is_enabled not working on rule changes
# v0.1.30
### Added
- Alerta alerter
- Added support for transitioning JIRA issues
- Option to recreate index in elastalert-create-index
### Fixed
- Update jira_ custom fields before each alert if they were modified
- Use json instead of simplejson
- Allow for relative path for smtp_auth_file
- Fixed some grammar issues
- Better code formatting of index mappings
- Better formatting and size limit for HipChat HTML
- Fixed gif link in readme for kibana plugin
- Fixed elastalert-test-rule with Elasticsearch > 4
- Added documentation for is_enabled option
## v0.1.29
###
### Added
- Added a feature forget_keys to prevent realerting when using flatline with query_key
- Added a new alert_text_type, aggregation_summary_only
......
......@@ -57,8 +57,59 @@ es_port: 9200
# This can be a unmapped index, but it is recommended that you run
# elastalert-create-index to set a mapping
writeback_index: elastalert_status
writeback_alias: elastalert_alerts
# If an alert fails for some reason, ElastAlert will retry
# sending the alert until this time period has elapsed
alert_time_limit:
days: 2
# Custom logging configuration
# If you want to setup your own logging configuration to log into
# files as well or to Logstash and/or modify log levels, use
# the configuration below and adjust to your needs.
# Note: if you run ElastAlert with --verbose/--debug, the log level of
# the "elastalert" logger is changed to INFO, if not already INFO/DEBUG.
#logging:
# version: 1
# incremental: false
# disable_existing_loggers: false
# formatters:
# logline:
# format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'
#
# handlers:
# console:
# class: logging.StreamHandler
# formatter: logline
# level: DEBUG
# stream: ext://sys.stderr
#
# file:
# class : logging.FileHandler
# formatter: logline
# level: DEBUG
# filename: elastalert.log
#
# loggers:
# elastalert:
# level: WARN
# handlers: []
# propagate: true
#
# elasticsearch:
# level: WARN
# handlers: []
# propagate: true
#
# elasticsearch.trace:
# level: WARN
# handlers: []
# propagate: true
#
# '': # root logger
# level: WARN
# handlers:
# - console
# - file
# propagate: false
......@@ -39,8 +39,10 @@ Currently, we have support built in for these alert types:
- HipChat
- Slack
- Telegram
- GoogleChat
- Debug
- Stomp
- theHive
Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types <writingrules>` and :ref:`Writing alerts <writingalerts>`)
......@@ -125,15 +127,18 @@ The environment variable ``ES_USE_SSL`` will override this field.
``es_password``: Optional; basic-auth password for connecting to ``es_host``. The environment variable ``ES_PASSWORD`` will override this field.
``es_url_prefix``: Optional; URL prefix for the Elasticsearch endpoint.
``es_url_prefix``: Optional; URL prefix for the Elasticsearch endpoint. The environment variable ``ES_URL_PREFIX`` will override this field.
``es_send_get_body_as``: Optional; Method for querying Elasticsearch - ``GET``, ``POST`` or ``source``. The default is ``GET``
``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``.
``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes.
Defaults to ``FileRulesLoader`` if not set.
``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all
files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload
or remove rules based on their respective config files.
or remove rules based on their respective config files. (only required when using ``FileRulesLoader``).
``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true``
......@@ -188,6 +193,25 @@ The default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in
``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch.
The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5.
``add_metadata_alert``: If set, alerts will include metadata described in rules (``category``, ``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``.
``skip_invalid``: If ``True``, skip invalid files instead of exiting.
=======
Logging
-------
By default, ElastAlert uses a simple basic logging configuration to print log messages to standard error.
You can change the log level to ``INFO`` messages by using the ``--verbose`` or ``--debug`` command line options.
If you need a more sophisticated logging configuration, you can provide a full logging configuration
in the config file. This way you can also configure logging to a file, to Logstash and
adjust the logging format.
For details, see the end of ``config.yaml.example`` where you can find an example logging
configuration.
.. _runningelastalert:
Running ElastAlert
......
......@@ -19,6 +19,7 @@ Contents:
recipes/adding_alerts
recipes/writing_filters
recipes/adding_enhancements
recipes/adding_loaders
recipes/signing_requests
Indices and Tables
......
.. _loaders:
Rules Loaders
========================
RulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to
gather rules for a particular source. Your RulesLoader needs to implement three member functions, and
will look something like this:
.. code-block:: python
class AwesomeNewRulesLoader(RulesLoader):
def get_names(self, conf, use_rule=None):
...
def get_hashes(self, conf, use_rule=None):
...
def get_yaml(self, rule):
...
You can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a
python module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``.
Example
-------
As an example loader, let's retrieve rules from a database rather than from the local file system. First, create a
modules folder for the loader in the ElastAlert directory.
.. code-block:: console
$ mkdir elastalert_modules
$ cd elastalert_modules
$ touch __init__.py
Now, in a file named ``mongo_loader.py``, add
.. code-block:: python
from pymongo import MongoClient
from elastalert.loaders import RulesLoader
import yaml
class MongoRulesLoader(RulesLoader):
def __init__(self, conf):
super(MongoRulesLoader, self).__init__(conf)
self.client = MongoClient(conf['mongo_url'])
self.db = self.client[conf['mongo_db']]
self.cache = {}
def get_names(self, conf, use_rule=None):
if use_rule:
return [use_rule]
rules = []
self.cache = {}
for rule in self.db.rules.find():
self.cache[rule['name']] = yaml.load(rule['yaml'])
rules.append(rule['name'])
return rules
def get_hashes(self, conf, use_rule=None):
if use_rule:
return [use_rule]
hashes = {}
self.cache = {}
for rule in self.db.rules.find():
self.cache[rule['name']] = rule['yaml']
hashes[rule['name']] = rule['hash']
return hashes
def get_yaml(self, rule):
if rule in self.cache:
return self.cache[rule]
self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml'])
return self.cache[rule]
Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the
default FileRulesLoader, so in your ``elastalert.conf`` file::
rules_loader: "elastalert_modules.mongo_loader.MongoRulesLoader"
......@@ -97,7 +97,7 @@ For ranges on fields::
Negation, and, or
*****************
Any of the filters can be embedded in ``not``, ``and``, and ``or``::
For Elasticsearch 2.X, any of the filters can be embedded in ``not``, ``and``, and ``or``::
filter:
- or:
......@@ -113,6 +113,13 @@ Any of the filters can be embedded in ``not``, ``and``, and ``or``::
term:
_type: "something"
For Elasticsearch 5.x, this will not work and to implement boolean logic use query strings::
filter:
- query:
query_string:
query: "somefield: somevalue OR foo: bar"
Loading Filters Directly From Kibana 3
--------------------------------------
......
This diff is collapsed.
......@@ -142,9 +142,9 @@ Running the ``elastalert-test-rule`` tool will test that your config file succes
$ elastalert-test-rule example_rules/example_frequency.yaml
If you want to specify a configuration file to use, you can run it with the config flag.
If you want to specify a configuration file to use, you can run it with the config flag::
$ elastalert-test-rule --config <path-to-config-file> example_rules/example_frequency.yaml.
$ elastalert-test-rule --config <path-to-config-file> example_rules/example_frequency.yaml
The configuration preferences will be loaded as follows:
1. Configurations specified in the yaml file.
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
# -*- coding: utf-8 -*-
from util import pretty_ts
class BaseEnhancement(object):
......@@ -14,6 +15,11 @@ class BaseEnhancement(object):
raise NotImplementedError()
class TimeEnhancement(BaseEnhancement):
def process(self, match):
match['@timestamp'] = pretty_ts(match['@timestamp'])
class DropMatchException(Exception):
""" ElastAlert will drop a match if this exception type is raised by an enhancement """
pass
{
"elastalert": {
"properties": {
"rule_name": {
"index": "not_analyzed",
"type": "string"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"alert_time": {
"type": "date",
"format": "dateOptionalTime"
},
"match_time": {
"type": "date",
"format": "dateOptionalTime"
},
"match_body": {
"type": "object",
"enabled": "false"
},
"aggregate_id": {
"index": "not_analyzed",
"type": "string"
}
}
}
}
{
"elastalert_error": {
"properties": {
"data": {
"type": "object",
"enabled": "false"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
{
"elastalert_status": {
"properties": {
"rule_name": {
"index": "not_analyzed",
"type": "string"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
{
"past_elastalert": {
"properties": {
"rule_name": {
"index": "not_analyzed",
"type": "string"
},
"match_body": {
"type": "object",
"enabled": "false"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"aggregate_id": {
"index": "not_analyzed",
"type": "string"
}
}
}
}
{
"silence": {
"properties": {
"rule_name": {
"index": "not_analyzed",
"type": "string"
},
"until": {
"type": "date",
"format": "dateOptionalTime"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
{
"properties": {
"rule_name": {
"type": "keyword"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"alert_time": {
"type": "date",
"format": "dateOptionalTime"
},
"match_time": {
"type": "date",
"format": "dateOptionalTime"
},
"match_body": {
"type": "object",
"enabled": "false"
},
"aggregate_id": {
"type": "keyword"
}
}
}
{
"properties": {
"data": {
"type": "object",
"enabled": "false"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
{
"properties": {
"rule_name": {
"type": "keyword"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
{
"properties": {
"rule_name": {
"type": "keyword"
},
"match_body": {
"type": "object",
"enabled": "false"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"aggregate_id": {
"type": "keyword"
}
}
}
{
"properties": {
"rule_name": {
"type": "keyword"
},
"until": {
"type": "date",
"format": "dateOptionalTime"
},
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
This diff is collapsed.
......@@ -15,18 +15,47 @@ class OpsGenieAlerter(Alerter):
def __init__(self, *args):
super(OpsGenieAlerter, self).__init__(*args)