From cf2a3f92b3e01ecaf56c93925aca248f3977d7a8 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Mon, 3 Dec 2018 11:46:36 -0800 Subject: [PATCH 1/4] Adding a `size` property to all job-reporting meta-data for consumption in ES/UI --- .../export_types/csv/server/execute_job.js | 5 +++-- .../lib/__tests__/max_size_string_builder.js | 14 ++++++++++++++ .../export_types/csv/server/lib/generate_csv.js | 3 ++- .../csv/server/lib/max_size_string_builder.js | 4 ++++ .../export_types/png/server/execute_job/index.js | 3 ++- .../printable_pdf/server/execute_job/index.js | 3 ++- 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js index 89ac80016fa559..4ef1658d88f8ee 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/execute_job.js +++ b/x-pack/plugins/reporting/export_types/csv/server/execute_job.js @@ -65,7 +65,7 @@ function executeJobFn(server) { const maxSizeBytes = config.get('xpack.reporting.csv.maxSizeBytes'); const scroll = config.get('xpack.reporting.csv.scroll'); - const { content, maxSizeReached } = await generateCsv({ + const { content, maxSizeReached, size } = await generateCsv({ searchRequest, fields, formatsMap, @@ -84,7 +84,8 @@ function executeJobFn(server) { return { content_type: 'text/csv', content, - max_size_reached: maxSizeReached + max_size_reached: maxSizeReached, + size, }; }; } diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js b/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js index d550399636845b..a4759ab12e0c8f 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js @@ -63,4 +63,18 @@ describe('MaxSizeStringBuilder', function () { expect(builder.getString()).to.be('a'); }); }); + + describe('getSizeInBytes', function () { + it(`should return 0 when no strings have been appended`, function () { + const builder = new MaxSizeStringBuilder(100); + expect(builder.getSizeInBytes).to.be(0); + }); + + it(`should the size in bytes`, function () { + const builder = new MaxSizeStringBuilder(100); + const stringValue = 'foobar'; + builder.tryAppend(stringValue); + expect(builder.getSizeInBytes).to.be(stringValue.lenth); + }); + }); }); diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js index 016081198d0334..b22c869427dd03 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js @@ -62,7 +62,8 @@ export function createGenerateCsv(logger) { logger('finished generating'); return { content: builder.getString(), - maxSizeReached + maxSizeReached, + size: builder.getSizeInBytes(), }; }; } diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/max_size_string_builder.js b/x-pack/plugins/reporting/export_types/csv/server/lib/max_size_string_builder.js index d1d86fd2bb250f..b4bdfcc13b3f6e 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/max_size_string_builder.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/max_size_string_builder.js @@ -22,6 +22,10 @@ export class MaxSizeStringBuilder { return false; } + getSizeInBytes() { + return this._size; + } + getString() { return this._buffer.slice(0, this._size).toString(); } diff --git a/x-pack/plugins/reporting/export_types/png/server/execute_job/index.js b/x-pack/plugins/reporting/export_types/png/server/execute_job/index.js index d9edbd7e0c6de5..fda79fb92da2a6 100644 --- a/x-pack/plugins/reporting/export_types/png/server/execute_job/index.js +++ b/x-pack/plugins/reporting/export_types/png/server/execute_job/index.js @@ -26,7 +26,8 @@ function executeJobFn(server) { }), map(buffer => ({ content_type: 'image/png', - content: buffer.toString('base64') + content: buffer.toString('base64'), + size: buffer.byteLength, })) ); diff --git a/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js b/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js index 4f5e6ca8306251..618a26b99cc1d2 100644 --- a/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js +++ b/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js @@ -34,7 +34,8 @@ function executeJobFn(server) { }), map(buffer => ({ content_type: 'application/pdf', - content: buffer.toString('base64') + content: buffer.toString('base64'), + size: buffer.byteLength, })) ); From 230d4768abd6ecc112e9c93dcae16562826382ea Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 4 Dec 2018 14:36:58 -0800 Subject: [PATCH 2/4] Adding a `size` field for reports, logging them and exposing them in report UI --- .../reporting/export_types/csv/server/lib/generate_csv.js | 5 +++-- .../reporting/public/components/report_info_button.tsx | 4 ++++ x-pack/plugins/reporting/public/lib/job_queue_client.ts | 5 ++++- .../reporting/server/lib/esqueue/helpers/create_index.js | 1 + x-pack/plugins/reporting/server/lib/esqueue/worker.js | 1 + 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js index b22c869427dd03..2d0133238bf82f 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/generate_csv.js @@ -58,12 +58,13 @@ export function createGenerateCsv(logger) { } finally { await iterator.return(); } + const size = builder.getSizeInBytes(); + logger(`finished generating, total size in bytes: ${size}`); - logger('finished generating'); return { content: builder.getString(), maxSizeReached, - size: builder.getSizeInBytes(), + size, }; }; } diff --git a/x-pack/plugins/reporting/public/components/report_info_button.tsx b/x-pack/plugins/reporting/public/components/report_info_button.tsx index 6168a2e2a8f6b2..56ad3f927498c0 100644 --- a/x-pack/plugins/reporting/public/components/report_info_button.tsx +++ b/x-pack/plugins/reporting/public/components/report_info_button.tsx @@ -126,6 +126,10 @@ export class ReportInfoButton extends Component { title: 'Content Type', description: get(info, 'output.content_type') || NA, }, + { + title: 'Size in Bytes', + description: get(info, 'output.size') || NA, + }, ], status: [ { diff --git a/x-pack/plugins/reporting/public/lib/job_queue_client.ts b/x-pack/plugins/reporting/public/lib/job_queue_client.ts index d5cdde62acfbd3..ca5d21dccfb1f4 100644 --- a/x-pack/plugins/reporting/public/lib/job_queue_client.ts +++ b/x-pack/plugins/reporting/public/lib/job_queue_client.ts @@ -27,7 +27,10 @@ export interface JobInfo { jobtype: string; created_by: string; timeout: number; - output: { content_type: string }; + output: { + content_type: string; + size: number; + }; process_expiration: string; completed_at: string; payload: { diff --git a/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js b/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js index bf022ba7f98f41..45be7529e68879 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js @@ -58,6 +58,7 @@ const schema = { type: 'object', properties: { content_type: { type: 'keyword' }, + size: { type: 'keyword' }, content: { type: 'object', enabled: false } } } diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index 02f9c054c2e3bd..97439b57135797 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -184,6 +184,7 @@ export class Worker extends events.EventEmitter { docOutput.content = output.content; docOutput.content_type = output.content_type || unknownMime; docOutput.max_size_reached = output.max_size_reached; + docOutput.size = output.size; } else { docOutput.content = output || defaultOutput; docOutput.content_type = unknownMime; From 8f0b983349c9c1edc557a160c3acba41e93b0d77 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Wed, 5 Dec 2018 11:42:30 -0800 Subject: [PATCH 3/4] Functions work the best when you actually call them --- .../csv/server/lib/__tests__/max_size_string_builder.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js b/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js index a4759ab12e0c8f..a688eb99918eeb 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js @@ -67,14 +67,14 @@ describe('MaxSizeStringBuilder', function () { describe('getSizeInBytes', function () { it(`should return 0 when no strings have been appended`, function () { const builder = new MaxSizeStringBuilder(100); - expect(builder.getSizeInBytes).to.be(0); + expect(builder.getSizeInBytes()).to.be(0); }); it(`should the size in bytes`, function () { const builder = new MaxSizeStringBuilder(100); const stringValue = 'foobar'; builder.tryAppend(stringValue); - expect(builder.getSizeInBytes).to.be(stringValue.lenth); + expect(builder.getSizeInBytes()).to.be(stringValue.lenth); }); }); }); From b75e64dd593a5b1fb1400765ac2729956c4cf508 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Wed, 5 Dec 2018 12:50:23 -0800 Subject: [PATCH 4/4] Spelling fixes --- .../csv/server/lib/__tests__/max_size_string_builder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js b/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js index a688eb99918eeb..15a10934b5db58 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/__tests__/max_size_string_builder.js @@ -74,7 +74,7 @@ describe('MaxSizeStringBuilder', function () { const builder = new MaxSizeStringBuilder(100); const stringValue = 'foobar'; builder.tryAppend(stringValue); - expect(builder.getSizeInBytes()).to.be(stringValue.lenth); + expect(builder.getSizeInBytes()).to.be(stringValue.length); }); }); });