From 968c575408625a44bf1f8ef47cb2297e17545b9e Mon Sep 17 00:00:00 2001 From: Rashid Khan Date: Tue, 3 Sep 2013 16:46:12 -0700 Subject: [PATCH 1/4] templated and scripted dashboards --- dashboards/logstash.js | 136 +++++++++++++++++++++++++++++++++++++ dashboards/logstash.json | 14 ++-- js/services.js | 94 +++++++++++++++++++++---- panels/histogram/module.js | 3 +- 4 files changed, 222 insertions(+), 25 deletions(-) create mode 100644 dashboards/logstash.js diff --git a/dashboards/logstash.js b/dashboards/logstash.js new file mode 100644 index 00000000000000..a2a811e7a09382 --- /dev/null +++ b/dashboards/logstash.js @@ -0,0 +1,136 @@ +/* Complex scripted Logstash dashboard */ + + +var dashboard, ARGS, queries; + +// arguments[0] contains a hash of the URL parameters, make it shorter +ARGS = arguments[0]; + +// Intialize a skeleton with nothing but a rows array and service object +dashboard = { + rows : [], + services : {} +}; + +// Set a title +dashboard.title = 'Logstash Search'; + +// Allow the user to set the index, if they dont, fall back to logstash. +if(!_.isUndefined(ARGS.index)) { + dashboard.index = { + default: ARGS.index, + interval: 'none' + } +} else { + dashboard.index = { + default: ARGS.index||'ADD_A_TIME_FILTER', + pattern: ARGS.pattern||'[logstash-]YYYY.MM.DD', + interval: ARGS.interval||'day' + } +} + +// In this dashboard we let users pass queries as comma seperated list to the query parameter. +// Or they can specify a split character using the split aparameter +// If query is defined, split it into a list of query objects +// NOTE: ids must be integers, hence the parseInt()s +if(!_.isUndefined(ARGS.query)) { + queries = _.object(_.map(ARGS.query.split(ARGS.split||','), function(v,k) { + return [k,{ + query: v, + id: parseInt(k), + alias: v + }]; + })); +} else { + // No queries passed? Initialize a single query to match everything + queries = { + 0: { + query: '*', + id: 0 + } + } +} + +// Now populate the query service with our objects +dashboard.services.query = { + list : queries, + ids : _.map(_.keys(queries),function(v){return parseInt(v);}) +} + +// Lets also add a default time filter, the value of which can be specified by the user +dashboard.services.filter = { + list: { + 0: { + from: kbn.time_ago(ARGS.from||'15m'), + to: new Date(), + field: ARGS.timefield||"@timestamp", + type: "time", + active: true, + id: 0 + } + }, + ids: [0] +} + +// Ok, lets make some rows. The Filters row is collapsed by default +dashboard.rows = [ + { + title: "Input", + height: "30px" + }, + { + title: "Filters", + height: "100px", + collapse: true + }, + { + title: "Chart", + height: "300px" + }, + { + title: "Events", + height: "400px" + } +]; + +// Setup some panels. A query panel and a filter panel on the same row +dashboard.rows[0].panels = [ + { + type: 'query', + span: 7 + }, + { + type: 'timepicker', + span: 5, + timespan: ARGS.from||'15m' + } +]; + +// Add a filtering panel to the 2nd row +dashboard.rows[1].panels = [ + { + type: 'filtering' + } +] + +// And a histogram that allows the user to specify the interval and time field +dashboard.rows[2].panels = [ + { + type: 'histogram', + time_field: ARGS.timefield||"@timestamp", + auto_int: true + } +] + +// And a table row where you can specify field and sort order +dashboard.rows[3].panels = [ + { + type: 'table', + fields: !_.isUndefined(ARGS.fields) ? ARGS.fields.split(',') : ['@timestamp','@message'], + sort: !_.isUndefined(ARGS.sort) ? ARGS.sort.split(',') : [ARGS.timefield||'@timestamp','desc'], + overflow: 'expand' + } +] + +// Now return the object and we're good! +return dashboard; \ No newline at end of file diff --git a/dashboards/logstash.json b/dashboards/logstash.json index 53ddc57d4a6cb8..ead4dcac67a86c 100644 --- a/dashboards/logstash.json +++ b/dashboards/logstash.json @@ -3,14 +3,11 @@ "services": { "query": { "idQueue": [ - 1, - 2, - 3, - 4 + 1 ], "list": { "0": { - "query": "*", + "query": "{{ARGS.query || '*'}}", "alias": "", "color": "#7EB26D", "id": 0 @@ -22,8 +19,7 @@ }, "filter": { "idQueue": [ - 1, - 2 + 1 ], "list": { "0": { @@ -70,7 +66,7 @@ "7d", "30d" ], - "timespan": "1h", + "timespan": "{{ARGS.from || '1h'}}", "timefield": "@timestamp", "timeformat": "", "refresh": { @@ -246,4 +242,4 @@ "pattern": "[logstash-]YYYY.MM.DD", "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED" } -} \ No newline at end of file +} diff --git a/js/services.js b/js/services.js index c254735882763b..ee94f352952d16 100644 --- a/js/services.js +++ b/js/services.js @@ -252,6 +252,13 @@ angular.module('kibana.services', []) ids : [], }); + // Defaults for query objects + var _query = { + query: '*', + alias: '', + pin: false, + type: 'lucene' + }; // For convenience var ejs = ejsResource(config.elasticsearch); var _q = dashboard.current.services.query; @@ -275,6 +282,12 @@ angular.module('kibana.services', []) self.list = dashboard.current.services.query.list; self.ids = dashboard.current.services.query.ids; + // Check each query object, populate its defaults + _.each(self.list,function(query,id) { + _.defaults(query,_query); + query.color = colorAt(id); + }); + if (self.ids.length === 0) { self.set({}); } @@ -290,16 +303,12 @@ angular.module('kibana.services', []) return false; } } else { - var _id = nextId(); - var _query = { - query: '*', - alias: '', - color: colorAt(_id), - pin: false, - id: _id, - type: 'lucene' - }; + var _id = query.id || nextId(); + query.id = _id; + query.color = query.color || colorAt(_id); _.defaults(query,_query); + + self.list[_id] = query; self.ids.push(_id); return _id; @@ -373,11 +382,13 @@ angular.module('kibana.services', []) .service('filterSrv', function(dashboard, ejsResource) { // Create an object to hold our service state on the dashboard dashboard.current.services.filter = dashboard.current.services.filter || {}; - _.defaults(dashboard.current.services.filter,{ + + // Defaults for it + var _d = { idQueue : [], list : {}, ids : [] - }); + }; // For convenience var ejs = ejsResource(config.elasticsearch); @@ -388,6 +399,9 @@ angular.module('kibana.services', []) // Call this whenever we need to reload the important stuff this.init = function() { + // Populate defaults + _.defaults(dashboard.current.services.filter,_d); + // Accessors self.list = dashboard.current.services.filter.list; self.ids = dashboard.current.services.filter.ids; @@ -606,6 +620,9 @@ angular.module('kibana.services', []) case ('file'): self.file_load(_id); break; + case('script'): + self.script_load(_id); + break; default: self.file_load('default.json'); } @@ -665,6 +682,7 @@ angular.module('kibana.services', []) }; this.dash_load = function(dashboard) { + // Cancel all timers timer.cancel_all(); @@ -744,11 +762,32 @@ angular.module('kibana.services', []) }; }; + var renderTemplate = function(json,params) { + var _r; + _.templateSettings = {interpolate : /\{\{(.+?)\}\}/g}; + var template = _.template(json); + var rendered = template({ARGS:params}); + + try { + _r = angular.fromJson(rendered); + } catch(e) { + _r = false; + } + return _r; + }; + this.file_load = function(file) { return $http({ url: "dashboards/"+file, method: "GET", + transformResponse: function(response) { + return renderTemplate(response,$routeParams); + } }).then(function(result) { + if(!result) { + return false; + } + var _dashboard = result.data; _.defaults(_dashboard,_dash); self.dash_load(_dashboard); @@ -759,11 +798,13 @@ angular.module('kibana.services', []) }); }; - this.elasticsearch_load = function(type,id) { return $http({ url: config.elasticsearch + "/" + config.kibana_index + "/"+type+"/"+id, - method: "GET" + method: "GET", + transformResponse: function(response) { + return renderTemplate(angular.fromJson(response)['_source']['dashboard'],$routeParams); + } }).error(function(data, status, headers, conf) { if(status === 0) { alertSrv.set('Error',"Could not contact Elasticsearch at "+config.elasticsearch+ @@ -774,7 +815,32 @@ angular.module('kibana.services', []) } return false; }).success(function(data, status, headers) { - self.dash_load(angular.fromJson(data['_source']['dashboard'])); + self.dash_load(data); + }); + }; + + this.script_load = function(file) { + return $http({ + url: "dashboards/"+file, + method: "GET", + transformResponse: function(response) { + /*jshint -W054 */ + var _f = new Function(response); + return _f($routeParams); + } + }).then(function(result) { + if(!result) { + return false; + } + var _dashboard = result.data; + _.defaults(_dashboard,_dash); + self.dash_load(_dashboard); + return true; + },function(result) { + alertSrv.set('Error', + "Could not load scripts/"+file+". Please make sure it exists and returns a valid dashboard" , + 'error'); + return false; }); }; diff --git a/panels/histogram/module.js b/panels/histogram/module.js index 7a410dd610fa62..5bdbb2235be234 100644 --- a/panels/histogram/module.js +++ b/panels/histogram/module.js @@ -135,8 +135,6 @@ angular.module('kibana.histogram', []) if(dashboard.indices.length === 0) { return; } - - var _range = $scope.get_time_range(); var _interval = $scope.get_interval(_range); @@ -177,6 +175,7 @@ angular.module('kibana.histogram', []) // Then run it var results = request.doSearch(); + // Populate scope when we have results results.then(function(results) { $scope.panelMeta.loading = false; From 7e5c36e9aa3ec67355207a51c83d29b0e50973f4 Mon Sep 17 00:00:00 2001 From: Rashid Khan Date: Wed, 4 Sep 2013 09:40:11 -0700 Subject: [PATCH 2/4] Made logstash.js look more like logstash.json. Added info notice for lack of failover and time filter --- dashboards/logstash.js | 59 +++++++++++++++++++++++++++++++++--------- js/services.js | 16 +++++++----- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/dashboards/logstash.js b/dashboards/logstash.js index a2a811e7a09382..eeb5107494712c 100644 --- a/dashboards/logstash.js +++ b/dashboards/logstash.js @@ -1,7 +1,27 @@ -/* Complex scripted Logstash dashboard */ +/* + * Complex scripted Logstash dashboard + * This script generates a dashboard object that Kibana can load. It also takes a number of user + * supplied URL parameters, none are required: + * + * index :: Which index to search? If this is specified, interval is set to 'none' + * pattern :: Does nothing if index is specified. Set a timestamped index pattern. Default: [logstash-]YYYY.MM.DD + * interval :: Sets the index interval (eg: day,week,month,year), Default: day + * + * split :: The character to split the queries on Default: ',' + * query :: By default, a comma seperated list of queries to run. Default: * + * + * from :: Search this amount of time back, eg 15m, 1h, 2d. Default: 15m + * timefield :: The field containing the time to filter on, Default: @timestamp + * + * fields :: comma seperated list of fields to show in the table + * sort :: comma seperated field to sort on, and direction, eg sort=@timestamp,desc + * + */ +var dashboard, ARGS, queries, _d_timespan; -var dashboard, ARGS, queries; +// Set a default timespan if one isn't specified +_d_timespan = '1h'; // arguments[0] contains a hash of the URL parameters, make it shorter ARGS = arguments[0]; @@ -22,6 +42,8 @@ if(!_.isUndefined(ARGS.index)) { interval: 'none' } } else { + // Don't fail to default + dashboard.failover = false; dashboard.index = { default: ARGS.index||'ADD_A_TIME_FILTER', pattern: ARGS.pattern||'[logstash-]YYYY.MM.DD', @@ -58,10 +80,11 @@ dashboard.services.query = { } // Lets also add a default time filter, the value of which can be specified by the user +// This isn't strictly needed, but it gets rid of the info alert about the missing time filter dashboard.services.filter = { list: { 0: { - from: kbn.time_ago(ARGS.from||'15m'), + from: kbn.time_ago(ARGS.from||_d_timespan), to: new Date(), field: ARGS.timefield||"@timestamp", type: "time", @@ -75,7 +98,11 @@ dashboard.services.filter = { // Ok, lets make some rows. The Filters row is collapsed by default dashboard.rows = [ { - title: "Input", + title: "Options", + height: "30px" + }, + { + title: "Query", height: "30px" }, { @@ -96,25 +123,33 @@ dashboard.rows = [ // Setup some panels. A query panel and a filter panel on the same row dashboard.rows[0].panels = [ { - type: 'query', - span: 7 + type: 'timepicker', + span: 6, + timespan: ARGS.from||_d_timespan }, { - type: 'timepicker', - span: 5, - timespan: ARGS.from||'15m' + type: 'dashcontrol', + span: 3 } ]; -// Add a filtering panel to the 2nd row +// Add a filtering panel to the 3rd row dashboard.rows[1].panels = [ + { + type: 'Query' + } +] + + +// Add a filtering panel to the 3rd row +dashboard.rows[2].panels = [ { type: 'filtering' } ] // And a histogram that allows the user to specify the interval and time field -dashboard.rows[2].panels = [ +dashboard.rows[3].panels = [ { type: 'histogram', time_field: ARGS.timefield||"@timestamp", @@ -123,7 +158,7 @@ dashboard.rows[2].panels = [ ] // And a table row where you can specify field and sort order -dashboard.rows[3].panels = [ +dashboard.rows[4].panels = [ { type: 'table', fields: !_.isUndefined(ARGS.fields) ? ARGS.fields.split(',') : ['@timestamp','@message'], diff --git a/js/services.js b/js/services.js index ee94f352952d16..66c70448c87ce8 100644 --- a/js/services.js +++ b/js/services.js @@ -659,9 +659,7 @@ angular.module('kibana.services', []) if(self.current.failover) { self.indices = [self.current.index.default]; } else { - alertSrv.set('No indices matched','The pattern '+self.current.index.pattern+ - ' did not match any indices in your selected'+ - ' time range.','info',5000); + // Do not issue refresh if no indices match. This should be removed when panels // properly understand when no indices are present return false; @@ -670,10 +668,14 @@ angular.module('kibana.services', []) $rootScope.$broadcast('refresh'); }); } else { - // This is not optimal, we should be getting the entire index list here, or at least every - // index that possibly matches the pattern - self.indices = [self.current.index.default]; - $rootScope.$broadcast('refresh'); + if(self.current.failover) { + self.indices = [self.current.index.default]; + $rootScope.$broadcast('refresh'); + } else { + alertSrv.set("No time filter", + 'Timestamped indices are configured without a failover. Waiting for time filter.', + 'info',5000); + } } } else { self.indices = [self.current.index.default]; From 0280626a1ed800340feeccf4dff2a5ca5cef2b0a Mon Sep 17 00:00:00 2001 From: Rashid Khan Date: Wed, 4 Sep 2013 10:55:48 -0700 Subject: [PATCH 3/4] Lint dashboard scripts --- Gruntfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index 1d640ff229bc08..3ff4a5200059bf 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -15,7 +15,7 @@ module.exports = function (grunt) { ' Licensed <%= pkg.license %> */\n\n' }, jshint: { - files: ['Gruntfile.js', 'js/*.js', 'panels/*/*.js' ], + files: ['Gruntfile.js', 'js/*.js', 'panels/*/*.js', 'dashboards/*.js' ], options: { jshintrc: '.jshintrc' } From 859c399646cff3911a630616a762acb1d55ea06c Mon Sep 17 00:00:00 2001 From: Rashid Khan Date: Wed, 4 Sep 2013 10:56:31 -0700 Subject: [PATCH 4/4] Namespace kibana url parameters --- dashboards/logstash.js | 35 +++++++++++++++++++---------------- js/app.js | 4 ++-- js/services.js | 8 ++++---- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/dashboards/logstash.js b/dashboards/logstash.js index eeb5107494712c..fb0edd2b3c7586 100644 --- a/dashboards/logstash.js +++ b/dashboards/logstash.js @@ -18,14 +18,17 @@ * */ -var dashboard, ARGS, queries, _d_timespan; +'use strict'; + +// Setup some variables +var dashboard, queries, _d_timespan; + +// All url parameters are available via the ARGS object +var ARGS; // Set a default timespan if one isn't specified _d_timespan = '1h'; -// arguments[0] contains a hash of the URL parameters, make it shorter -ARGS = arguments[0]; - // Intialize a skeleton with nothing but a rows array and service object dashboard = { rows : [], @@ -40,7 +43,7 @@ if(!_.isUndefined(ARGS.index)) { dashboard.index = { default: ARGS.index, interval: 'none' - } + }; } else { // Don't fail to default dashboard.failover = false; @@ -48,7 +51,7 @@ if(!_.isUndefined(ARGS.index)) { default: ARGS.index||'ADD_A_TIME_FILTER', pattern: ARGS.pattern||'[logstash-]YYYY.MM.DD', interval: ARGS.interval||'day' - } + }; } // In this dashboard we let users pass queries as comma seperated list to the query parameter. @@ -59,7 +62,7 @@ if(!_.isUndefined(ARGS.query)) { queries = _.object(_.map(ARGS.query.split(ARGS.split||','), function(v,k) { return [k,{ query: v, - id: parseInt(k), + id: parseInt(k,10), alias: v }]; })); @@ -70,14 +73,14 @@ if(!_.isUndefined(ARGS.query)) { query: '*', id: 0 } - } + }; } // Now populate the query service with our objects dashboard.services.query = { list : queries, - ids : _.map(_.keys(queries),function(v){return parseInt(v);}) -} + ids : _.map(_.keys(queries),function(v){return parseInt(v,10);}) +}; // Lets also add a default time filter, the value of which can be specified by the user // This isn't strictly needed, but it gets rid of the info alert about the missing time filter @@ -93,7 +96,7 @@ dashboard.services.filter = { } }, ids: [0] -} +}; // Ok, lets make some rows. The Filters row is collapsed by default dashboard.rows = [ @@ -138,7 +141,7 @@ dashboard.rows[1].panels = [ { type: 'Query' } -] +]; // Add a filtering panel to the 3rd row @@ -146,7 +149,7 @@ dashboard.rows[2].panels = [ { type: 'filtering' } -] +]; // And a histogram that allows the user to specify the interval and time field dashboard.rows[3].panels = [ @@ -155,7 +158,7 @@ dashboard.rows[3].panels = [ time_field: ARGS.timefield||"@timestamp", auto_int: true } -] +]; // And a table row where you can specify field and sort order dashboard.rows[4].panels = [ @@ -165,7 +168,7 @@ dashboard.rows[4].panels = [ sort: !_.isUndefined(ARGS.sort) ? ARGS.sort.split(',') : [ARGS.timefield||'@timestamp','desc'], overflow: 'expand' } -] +]; // Now return the object and we're good! -return dashboard; \ No newline at end of file +return dashboard; diff --git a/js/app.js b/js/app.js index b812468c198cd9..7a1903381c00ac 100644 --- a/js/app.js +++ b/js/app.js @@ -48,10 +48,10 @@ labjs.wait(function(){ .when('/dashboard', { templateUrl: 'partials/dashboard.html', }) - .when('/dashboard/:type/:id', { + .when('/dashboard/:kbnType/:kbnId', { templateUrl: 'partials/dashboard.html', }) - .when('/dashboard/:type/:id/:params', { + .when('/dashboard/:kbnType/:kbnId/:params', { templateUrl: 'partials/dashboard.html' }) .otherwise({ diff --git a/js/services.js b/js/services.js index 66c70448c87ce8..372460d4a3b8be 100644 --- a/js/services.js +++ b/js/services.js @@ -606,9 +606,9 @@ angular.module('kibana.services', []) var route = function() { // Is there a dashboard type and id in the URL? - if(!(_.isUndefined($routeParams.type)) && !(_.isUndefined($routeParams.id))) { - var _type = $routeParams.type; - var _id = $routeParams.id; + if(!(_.isUndefined($routeParams.kbnType)) && !(_.isUndefined($routeParams.kbnId))) { + var _type = $routeParams.kbnType; + var _id = $routeParams.kbnId; switch(_type) { case ('elasticsearch'): @@ -827,7 +827,7 @@ angular.module('kibana.services', []) method: "GET", transformResponse: function(response) { /*jshint -W054 */ - var _f = new Function(response); + var _f = new Function("ARGS",response); return _f($routeParams); } }).then(function(result) {