Skip to content

Commit

Permalink
Extract Kibana dashboards (#7224)
Browse files Browse the repository at this point in the history 
Currently when a PR is opened to change or add a dashboard the PR's are hard to review even if only a small detail was changed. The reasons is that the Kibana json objects contain json as string. The content inside these strings is valid JSON if it is decoded.

With this PR the dashboards in Metricbeat are modified that they contain the full decoded JSON objects instead of the string. Additional the JSON entries are sorted. This makes also the content of the visualisations readable, will create minimal diffs and allows to even apply small fixes on the code base. In addition we can now validate if it's valid JSON and introduce automatic scripts to remove potentially unneeded fields from the dashboards to clean them up.

All the existing dashboards were decoded with the following command:
```
python ../libbeat/scripts/unpack_dashboards.py --transform=decode --glob="/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/metricbeat/module/*/_meta/kibana/6/dashboard/*.json"
```

This command has to be applied to dashboards exported from Kibana before adding them to the module directory.

To make sure the import still works as before, on collection of the dashboard they are converted back into the encoded format.

This change currently only applies to Metricbeat dashboards but could be applied to all.
  • Loading branch information
@ruflin @jsoriano
ruflin authored and jsoriano committed Jun 4, 2018
1 parent ccd36fb commit 2e1b7c8
Show file tree
Hide file tree
Showing 24 changed files with 14,570 additions and 2,678 deletions.
53 changes: 53 additions & 0 deletions libbeat/scripts/unpack_dashboards.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import json
import sys
import glob
import argparse


def transform_data(data, method):
for obj in data["objects"]:
if "uiStateJSON" in obj["attributes"]:
obj["attributes"]["uiStateJSON"] = method(obj["attributes"]["uiStateJSON"])

if "optionsJSON" in obj["attributes"]:
obj["attributes"]["optionsJSON"] = method(obj["attributes"]["optionsJSON"])

if "panelsJSON" in obj["attributes"]:
obj["attributes"]["panelsJSON"] = method(obj["attributes"]["panelsJSON"])

if "visState" in obj["attributes"]:
obj["attributes"]["visState"] = method(obj["attributes"]["visState"])

if "kibanaSavedObjectMeta" in obj["attributes"] and "searchSourceJSON" in obj["attributes"]["kibanaSavedObjectMeta"]:
obj["attributes"]["kibanaSavedObjectMeta"]["searchSourceJSON"] = method(
obj["attributes"]["kibanaSavedObjectMeta"]["searchSourceJSON"])


def transform_file(path, method):
with open(path) as f:
data = json.load(f)

transform_data(data, method)
return data


if __name__ == "__main__":

parser = argparse.ArgumentParser(description="Convert dashboards")
parser.add_argument("--transform", help="Decode or encode", default="encode")
parser.add_argument("--glob", help="Glob pattern")

args = parser.parse_args()

paths = glob.glob(args.glob)

method = json.dumps
if args.transform == "decode":
method = json.loads

for path in paths:
data = transform_file(path, method)
new_data = json.dumps(data, sort_keys=True, indent=4)

with open(path, 'w') as f:
f.write(new_data)
3 changes: 3 additions & 0 deletions metricbeat/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ kibana:
@rm -rf _meta/kibana
@mkdir -p _meta/kibana
@-cp -r module/*/_meta/kibana _meta/
@-cp -r module/*/_meta/kibana _meta/
@# Convert all dashboards to string
@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana/6/dashboard/*.json"

# Collects all module docs
.PHONY: collect-docs
Expand Down
935 changes: 780 additions & 155 deletions metricbeat/module/apache/_meta/kibana/6/dashboard/Metricbeat-apache-overview.json
View file

Large diffs are not rendered by default.

1,201 changes: 1,015 additions & 186 deletions metricbeat/module/docker/_meta/kibana/6/dashboard/Metricbeat-docker-overview.json
View file

Large diffs are not rendered by default.

408 changes: 296 additions & 112 deletions metricbeat/module/golang/_meta/kibana/6/dashboard/Metricbeat-golang-overview.json
View file

Large diffs are not rendered by default.

135 changes: 113 additions & 22 deletions metricbeat/module/haproxy/_meta/kibana/6/dashboard/Metricbeat-haproxy-backend.json
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,114 @@
{
"objects": [
{
"attributes": {
"description": "",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
},
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":6,\"h\":2,\"i\":\"1\"},\"id\":\"a64b4fd0-471c-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":6,\"y\":0,\"w\":3,\"h\":2,\"i\":\"2\"},\"id\":\"794b6cd0-471d-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":6,\"y\":2,\"w\":6,\"h\":4,\"i\":\"3\"},\"id\":\"bb0ab500-4735-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":9,\"y\":0,\"w\":3,\"h\":2,\"i\":\"4\"},\"id\":\"40bed190-473b-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":0,\"y\":2,\"w\":6,\"h\":2,\"i\":\"5\"},\"id\":\"0751ed00-479c-11e8-bc13-1397384faad3\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":4,\"w\":6,\"h\":2,\"i\":\"6\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"b3463670-47a1-11e8-bc13-1397384faad3\"}]",
"timeRestore": false,
"title": "[Metricbeat Haproxy] Backend",
"version": 1
},
"id": "9151c900-471d-11e8-bc13-1397384faad3",
"type": "dashboard",
"updated_at": "2018-04-24T18:31:25.838Z",
"version": 15
}
],
"version": "6.2.2"
}
"objects": [
{
"attributes": {
"description": "",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"filter": [],
"highlightAll": true,
"query": {
"language": "lucene",
"query": ""
},
"version": true
}
},
"optionsJSON": {
"darkTheme": false,
"hidePanelTitles": false,
"useMargins": true
},
"panelsJSON": [
{
"gridData": {
"h": 2,
"i": "1",
"w": 6,
"x": 0,
"y": 0
},
"id": "a64b4fd0-471c-11e8-bc13-1397384faad3",
"panelIndex": "1",
"type": "visualization",
"version": "6.2.2"
},
{
"gridData": {
"h": 2,
"i": "2",
"w": 3,
"x": 6,
"y": 0
},
"id": "794b6cd0-471d-11e8-bc13-1397384faad3",
"panelIndex": "2",
"type": "visualization",
"version": "6.2.2"
},
{
"gridData": {
"h": 4,
"i": "3",
"w": 6,
"x": 6,
"y": 2
},
"id": "bb0ab500-4735-11e8-bc13-1397384faad3",
"panelIndex": "3",
"type": "visualization",
"version": "6.2.2"
},
{
"gridData": {
"h": 2,
"i": "4",
"w": 3,
"x": 9,
"y": 0
},
"id": "40bed190-473b-11e8-bc13-1397384faad3",
"panelIndex": "4",
"type": "visualization",
"version": "6.2.2"
},
{
"gridData": {
"h": 2,
"i": "5",
"w": 6,
"x": 0,
"y": 2
},
"id": "0751ed00-479c-11e8-bc13-1397384faad3",
"panelIndex": "5",
"type": "visualization",
"version": "6.2.2"
},
{
"gridData": {
"h": 2,
"i": "6",
"w": 6,
"x": 0,
"y": 4
},
"id": "b3463670-47a1-11e8-bc13-1397384faad3",
"panelIndex": "6",
"type": "visualization",
"version": "6.2.2"
}
],
"timeRestore": false,
"title": "[Metricbeat Haproxy] Backend",
"version": 1
},
"id": "9151c900-471d-11e8-bc13-1397384faad3",
"type": "dashboard",
"updated_at": "2018-04-24T18:31:25.838Z",
"version": 15
}
],
"version": "6.2.2"
}
83 changes: 61 additions & 22 deletions metricbeat/module/haproxy/_meta/kibana/6/dashboard/Metricbeat-haproxy-frontend.json
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,62 @@
{
"objects": [
{
"attributes": {
"description": "",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
},
"optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}",
"panelsJSON": "[{\"panelIndex\":\"2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":6,\"h\":3,\"i\":\"2\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"a64b4fd0-471c-11e8-bc13-1397384faad3\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":6,\"y\":0,\"w\":6,\"h\":3,\"i\":\"3\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"86159190-47c5-11e8-bc13-1397384faad3\"}]",
"timeRestore": false,
"title": "[Metricbeat Haproxy] Frontend",
"version": 1
},
"id": "d5878d00-47c5-11e8-bc13-1397384faad3",
"type": "dashboard",
"updated_at": "2018-04-24T18:32:51.945Z",
"version": 5
}
],
"version": "6.2.2"
}
"objects": [
{
"attributes": {
"description": "",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"filter": [],
"highlightAll": true,
"query": {
"language": "lucene",
"query": ""
},
"version": true
}
},
"optionsJSON": {
"darkTheme": false,
"hidePanelTitles": false,
"useMargins": true
},
"panelsJSON": [
{
"gridData": {
"h": 3,
"i": "2",
"w": 6,
"x": 0,
"y": 0
},
"id": "a64b4fd0-471c-11e8-bc13-1397384faad3",
"panelIndex": "2",
"type": "visualization",
"version": "6.2.2"
},
{
"gridData": {
"h": 3,
"i": "3",
"w": 6,
"x": 6,
"y": 0
},
"id": "86159190-47c5-11e8-bc13-1397384faad3",
"panelIndex": "3",
"type": "visualization",
"version": "6.2.2"
}
],
"timeRestore": false,
"title": "[Metricbeat Haproxy] Frontend",
"version": 1
},
"id": "d5878d00-47c5-11e8-bc13-1397384faad3",
"type": "dashboard",
"updated_at": "2018-04-24T18:32:51.945Z",
"version": 5
}
],
"version": "6.2.2"
}
Loading

0 comments on commit 2e1b7c8

Please sign in to comment.