GIS Map in Elasticsearch with Kibana using Vega, Scripted Fields & Painless

Elasticsearch is a distributed open source, RESTful search engine built on top of Apache Lucene and released under an Apache license. Kibana is an open source data visualization plugin for Elasticsearch. Vega (and Vega-lite) allows to beyond the built-in visualizations offered by Kibana.

In this short tutorial we will use Vega to create a GIS map that displays individual documents in Elasticsearch into a Kibana map as marks, that have different shapes and colors, with information about the documents on the marks.

The graph will look like

GIS Map using Vega

{
"$schema": "https://vega.github.io/schema/vega/v3.0.json",
config: {
kibana: {
type: map
latitude: 15
longitude: -10
zoom: 3
}
}
"data": [
{
"name": "observations",
"url": {
index: logstash-2018*
"body": {
"size": 10000
_source: {
includes: ["@timestamp", "items_per_minute", "school", "geoip.location", "county", "subcounty","zone", "lesson_start_time","subject", "end_time", "start_time", "class"]
}
script_fields : {
lesson_duration : {
script : {
lang: 'painless',
source: doc['end_time'].date.minuteOfDay - doc['start_time'].date.minuteOfDay;
}
}
lesson_date : {
script : {
lang: 'painless',
// Get the time in the correct format
source: doc['start_time'].date.yearOfEra + '-' + doc['start_time'].date.monthOfYear + '-' + doc['start_time'].date.dayOfMonth
}
}
grade : {
script : {
lang: 'painless',
"source": "if(doc.containsKey('formId.keyword')){if(doc['formId.keyword'].value == 'Gradethreeobservationtool'){return 3;}if(doc['formId.keyword'].value == 'maths-grade3'){return 3;}if(doc['formId.keyword'].value == 'class-12-lesson-observation-with-pupil-books'){if(doc.containsKey('class.keyword')){return doc['class.keyword'].value;}}if(doc['formId.keyword'].value == 'maths-teachers-observation-tool'){if(doc.containsKey('class.keyword')){return doc['class.keyword'].value;}}}"
}
}
}
"sort": {"start_time":"desc"}
"query": {
"bool": {
"must": [
// This string will be replaced with the auto-generated "MUST" clause
"%dashboard_context-must_clause%",
// apply timefilter (upper right corner) to the @timestamp variable
{
"range": {
"@timestamp": {
// "%timefilter%" will be replaced with the current
// values of the time filter (from the upper right corner)
"%timefilter%": true
// Only work with %timefilter%
// Shift the current timefilter by 10 units back
"shift": 10,
// supports week, day (default), hour, minute, second.
"unit": "minute"
}
}
}
{"exists": {"field": "geoip.location"}}
{"exists": {"field": "school"}}
{"exists": {"field": "start_time"}}
],
"must_not": [
// This string will be replaced with the auto-generated "MUST-NOT" clause
"%dashboard_context-must_not_clause%"
{"match": { "school": "orphaned" }}
]
}
}
}}
"format": { "type": "json", "property": "hits.hits"}
"transform": [
{
"lookup": "geoip_location",
"type": "geopoint",
"projection": "projection",
"fields": [
_source.geoip\.location.lon
_source.geoip\.location.lat
]
}
]
}
{
"name": "selected",
"source": "observations",
"transform": [
{
"type": "filter",
"expr": "datum === selected"
}
]
}
]
"signals": [
{
"name": "selected",
"value": null,
"on": [
{"events": "symbol:mouseover", "update": "datum"},
{"events": "symbol:mouseout", "update": "null"}
]
}
],
"marks": [
{
name: "observation"
type: "symbol"
from: {data: "observations"}
encode: {
"enter": {
// different shapes for grades
"shape": [
{"test": "datum.fields.grade == 1", "value": "square"}
{"test": "datum.fields.grade == 2", "value": "triangle-up"}
{"value": "circle"}
]
// different colors for subjects
"fill": [
{"test": "datum._source.subject === 'English'", "value": "#159"}
{"test": "datum._source.subject === 'Maths'", "value": "#195"}
{"value": "#815"}
]
"size": {"value": "200"}
},
update: {
xc: {signal: "datum.x"}
yc: {signal: "datum.y"}
tooltip: {
signal: "{School: datum._source.school, County: datum._source.county, Subcounty: datum._source.subcounty, Zone: datum._source.zone, Subject: datum._source.subject, Grade:''+ datum.fields.grade, 'Items Per Minute': datum._source.items_per_minute, Date: ''+datum.fields.lesson_date, 'Start Time': hours(datum._source.start_time) + 3 + ':' + minutes(datum._source.start_time), 'End Time': hours(datum._source.end_time) + 3 + ':' + minutes(datum._source.end_time) + ' (' + datum.fields.lesson_duration + ' minutes)'}"
}
"fillOpacity": {"value": 0.7}
}
"hover": {
"fillOpacity": {"value": 0.3}
}
}
}
]
}