Add support for Elasticsearch datasource #99 (#100)

* add elasticsearch Target class and helpers

Add an ElasticsearchTarget and query related helper classes to create
graphs with queries from an elasticsearch datasource.

This commit only adds partial support for elasticsearch queries.
Grafana supports a lot more Metric and Aggregators for elasticsearch.

* docs: add an elasticsearch dashboard example

* elasticsearch: adapt implementation to review comments

- add documentation
- improve naming
- remove Settings objects, move the fields to the aggregator objects
- make fields that don't change constants in to_json_data()

Thanks @fho!

Author: fho

docs/example-elasticsearch.dashboard.py

@@ -0,0 +1,87 @@
+"""
+This is an exemplary Grafana board that uses an elasticsearch datasource.
+
+The graph shows the following metrics for HTTP requests to the URL path "/login":
+- number of successful requests resulted in a HTTP response code between 200-300
+- number of failed requests resulted in a HTTP response code between 400-500,
+- Max. response time per point of time of HTTP requests
+"""
+
+from grafanalib.core import *
+from grafanalib.elasticsearch import *
+
+suc_label = "Success (200-300)"
+clt_err_label = "Client Errors (400-500)"
+resptime_label = "Max response time"
+
+filters = [
+    Filter(query="response: [200 TO 300]", label=suc_label),
+    Filter(query="response: [400 TO 500]", label=clt_err_label),
+]
+
+tgts = [
+    ElasticsearchTarget(
+        query='request: "/login"',
+        bucketAggs=[
+            FiltersGroupBy(filters=filters),
+            DateHistogramGroupBy(interval="10m")],
+    ).auto_bucket_agg_ids(),
+    ElasticsearchTarget(
+        query='request: "/login"',
+        metricAggs=[MaxMetricAgg(field="resptime")],
+        alias=resptime_label,
+    ).auto_bucket_agg_ids(),
+]
+
+g = Graph(
+    title="login requests",
+    dataSource="elasticsearch",
+    targets=tgts,
+    lines=False,
+    legend=Legend(alignAsTable=True, rightSide=True, total=True, current=True, max=True),
+    lineWidth=1,
+    nullPointMode=NULL_AS_NULL,
+    seriesOverrides=[
+        {
+            "alias": suc_label,
+            "bars": True,
+            "lines": False,
+            "stack": "A",
+            "yaxis": 1,
+            "color": "#629E51"
+        },
+        {
+            "alias": clt_err_label,
+            "bars": True,
+            "lines": False,
+            "stack": "A",
+            "yaxis": 1,
+            "color": "#E5AC0E"
+        },
+        {
+            "alias": resptime_label,
+            "lines": True,
+            "fill": 0,
+            "nullPointMode": "connected",
+            "steppedLine": True,
+            "yaxis": 2,
+            "color": "#447EBC"
+        },
+    ],
+    yAxes=[
+        YAxis(
+            label="Count",
+            format=SHORT_FORMAT,
+            decimals=0
+        ),
+        YAxis(
+            label="Response Time",
+            format=SECONDS_FORMAT,
+            decimals=2
+        ),
+    ],
+    transparent=True,
+    span=12,
+)
+
+dashboard = Dashboard(title="HTTP dashboard", rows=[Row(panels=[g])])

grafanalib/elasticsearch.py

@@ -0,0 +1,207 @@
+"""Helpers to create Elasticsearch-specific Grafana queries."""
+
+import attr
+import itertools
+from attr.validators import instance_of
+
+DATE_HISTOGRAM_DEFAULT_FIELD = "time_iso8601"
+ORDER_ASC = "asc"
+ORDER_DESC = "desc"
+
+
+@attr.s
+class CountMetricAgg(object):
+    """An aggregator that counts the number of values.
+
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-valuecount-aggregation.html
+
+    It's the default aggregator for elasticsearch queries.
+    """
+    def to_json_data(self):
+        return {
+            'type': 'count',
+            'field': 'select field',
+            'settings': {},
+        }
+
+
+@attr.s
+class MaxMetricAgg(object):
+    """An aggregator that provides the max. value among the values.
+
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-max-aggregation.html
+
+    :param field: name of elasticsearch field to provide the maximum for
+    """
+    field = attr.ib(default="", validator=instance_of(str))
+
+    def to_json_data(self):
+        return {
+            'type': 'max',
+            'field': self.field,
+            'settings': {},
+        }
+
+
+@attr.s
+class DateHistogramGroupBy(object):
+    """A bucket aggregator that groups results by date.
+
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html
+
+    :param id: ascending unique number per GroupBy clause
+    :param field: name of the elasticsearch field to group by
+    :param interval: interval to group by
+    :param minDocCount: min. amount of records in the timespan to return a
+                        result
+    """
+    id = attr.ib(default=0, validator=instance_of(int))
+    field = attr.ib(default=DATE_HISTOGRAM_DEFAULT_FIELD,
+                    validator=instance_of(str))
+    interval = attr.ib(default="auto", validator=instance_of(str))
+    minDocCount = attr.ib(default=0, validator=instance_of(int))
+
+    def to_json_data(self):
+        return {
+                'field': self.field,
+                'id': str(self.id),
+                'settings': {
+                             'interval': self.interval,
+                             'min_doc_count': self.minDocCount,
+                             'trimEdges': 0,
+                             },
+                'type': 'date_histogram',
+                }
+
+
+@attr.s
+class Filter(object):
+    """ A Filter for a FilterGroupBy aggregator.
+
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-filter-aggregation.html
+
+    :param label: label for the metric that is shown in the graph
+    :param query: the query to filter by
+    """
+    label = attr.ib(default="", validator=instance_of(str))
+    query = attr.ib(default="", validator=instance_of(str))
+
+    def to_json_data(self):
+        return {
+                'label': self.label,
+                'query': self.query,
+                }
+
+
+@attr.s
+class FiltersGroupBy(object):
+    """ A bucket aggregator that groups records by a filter expression.
+
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-filter-aggregation.html
+
+    :param id: ascending unique number per GroupBy clause
+    :param filters: list of Filter objects
+    """
+    id = attr.ib(default=0, validator=instance_of(int))
+    filters = attr.ib(default=attr.Factory(list))
+
+    def to_json_data(self):
+        return {
+                'id': str(self.id),
+                'settings': {
+                             'filters': self.filters,
+                             },
+                'type': 'filters',
+                }
+
+
+@attr.s
+class TermsGroupBy(object):
+    """ A multi-bucket aggregator based on field values.
+
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html
+
+    :param id: ascending unique number per GroupBy clause
+    :param field: name of the field to group by
+    :param minDocCount: min. amount of matching records to return a result
+    :param order: ORDER_ASC or ORDER_DESC
+    :param orderBy: term to order the bucker
+    :param size: how many buckets are returned
+    """
+    field = attr.ib(validator=instance_of(str))
+    id = attr.ib(default=0, validator=instance_of(int))
+    minDocCount = attr.ib(default=1, validator=instance_of(int))
+    order = attr.ib(default=ORDER_DESC, validator=instance_of(str))
+    orderBy = attr.ib(default="_term", validator=instance_of(str))
+    size = attr.ib(default=0, validator=instance_of(int))
+
+    def to_json_data(self):
+        return {
+                'id': str(self.id),
+                'type': 'terms',
+                'field': self.field,
+                'settings': {
+                    'min_doc_count': self.minDocCount,
+                    'order': self.order,
+                    'order_by': self.orderBy,
+                    'size': self.size,
+                 },
+                }
+
+
+@attr.s
+class ElasticsearchTarget(object):
+    """Generates Elasticsearch target JSON structure.
+
+    Grafana docs on using Elasticsearch:
+    http://docs.grafana.org/features/datasources/elasticsearch/
+    Elasticsearch docs on querying or reading data:
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
+
+    :param alias: legend alias
+    :param bucketAggs: Bucket aggregators
+    :param metricAggs: Metric Aggregators
+    :param query: query
+    :param refId: target reference id
+    """
+
+    alias = attr.ib(default=None)
+    bucketAggs = attr.ib(default=attr.Factory(
+                                          lambda: [DateHistogramGroupBy()]))
+    metricAggs = attr.ib(default=attr.Factory(lambda: [CountMetricAgg()]))
+    query = attr.ib(default="", validator=instance_of(str))
+    refId = attr.ib(default="", validator=instance_of(str))
+
+    def _map_bucket_aggs(self, f):
+        return attr.assoc(self, bucketAggs=list(map(f, self.bucketAggs)))
+
+    def auto_bucket_agg_ids(self):
+        """Give unique IDs all bucketAggs without ID.
+
+        Returns a new ``ElasticsearchTarget`` that is the same as this one,
+        except all of the bucketAggs have their ``id`` property set. Any panels
+        which had an ``id`` property set will keep that property, all others
+        will have auto-generated IDs provided for them.
+
+        If the bucketAggs don't have unique ID associated with it, the
+        generated graph will be broken.
+        """
+        ids = set([agg.id for agg in self.bucketAggs if agg.id])
+        auto_ids = (i for i in itertools.count(1) if i not in ids)
+
+        def set_id(agg):
+            if agg.id:
+                return agg
+
+            return attr.evolve(agg, id=next(auto_ids))
+
+        return self._map_bucket_aggs(set_id)
+
+    def to_json_data(self):
+        return {
+            'alias': self.alias,
+            'bucketAggs': self.bucketAggs,
+            'metrics': self.metricAggs,
+            'query': self.query,
+            'refId': self.refId,
+        }