From 3665dfb555cb546023f28646c438962d085bb227 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Fri, 10 Apr 2020 15:45:33 -0700 Subject: [PATCH] Adds a new config flag to encode with BOM for our CSVs (#63006) (#63265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds a new config flag to encode with BOM for our CSVs * Push out bom-chars to it's own constant * Getting those snapshots back into shape 💪 Co-authored-by: Elastic Machine Co-authored-by: Joel Griffith Co-authored-by: Elastic Machine --- .../__snapshots__/index.test.js.snap | 4 ++ .../plugins/reporting/common/constants.ts | 1 + x-pack/legacy/plugins/reporting/config.ts | 1 + .../csv/server/execute_job.test.js | 45 +++++++++++++++++++ .../export_types/csv/server/execute_job.ts | 6 ++- .../plugins/reporting/server/config/index.ts | 1 + 6 files changed, 56 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/__snapshots__/index.test.js.snap b/x-pack/legacy/plugins/reporting/__snapshots__/index.test.js.snap index 757677f1d4f82..3ae3079da136b 100644 --- a/x-pack/legacy/plugins/reporting/__snapshots__/index.test.js.snap +++ b/x-pack/legacy/plugins/reporting/__snapshots__/index.test.js.snap @@ -66,6 +66,7 @@ Object { "duration": "30s", "size": 500, }, + "useByteOrderMarkEncoding": false, }, "enabled": true, "encryptionKey": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -162,6 +163,7 @@ Object { "duration": "30s", "size": 500, }, + "useByteOrderMarkEncoding": false, }, "enabled": true, "index": ".reporting", @@ -257,6 +259,7 @@ Object { "duration": "30s", "size": 500, }, + "useByteOrderMarkEncoding": false, }, "enabled": true, "encryptionKey": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -353,6 +356,7 @@ Object { "duration": "30s", "size": 500, }, + "useByteOrderMarkEncoding": false, }, "enabled": true, "index": ".reporting", diff --git a/x-pack/legacy/plugins/reporting/common/constants.ts b/x-pack/legacy/plugins/reporting/common/constants.ts index 8f7a06ba9f8e9..e3d6a4274e7df 100644 --- a/x-pack/legacy/plugins/reporting/common/constants.ts +++ b/x-pack/legacy/plugins/reporting/common/constants.ts @@ -19,6 +19,7 @@ export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv export const CONTENT_TYPE_CSV = 'text/csv'; export const CSV_REPORTING_ACTION = 'downloadCsvReport'; +export const CSV_BOM_CHARS = '\ufeff'; export const WHITELISTED_JOB_CONTENT_TYPES = [ 'application/json', diff --git a/x-pack/legacy/plugins/reporting/config.ts b/x-pack/legacy/plugins/reporting/config.ts index 211fa70301bbf..5eceb84c83e43 100644 --- a/x-pack/legacy/plugins/reporting/config.ts +++ b/x-pack/legacy/plugins/reporting/config.ts @@ -135,6 +135,7 @@ export async function config(Joi: any) { .default(), }).default(), csv: Joi.object({ + useByteOrderMarkEncoding: Joi.boolean().default(false), checkForFormulas: Joi.boolean().default(true), enablePanelActionDownload: Joi.boolean().default(true), maxSizeBytes: Joi.number() diff --git a/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.test.js b/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.test.js index 3ea1bc283983d..2e4aa4383ec9c 100644 --- a/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.test.js +++ b/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.test.js @@ -13,6 +13,7 @@ import { createMockReportingCore } from '../../../test_helpers'; import { LevelLogger } from '../../../server/lib/level_logger'; import { setFieldFormats } from '../../../server/services'; import { executeJobFactory } from './execute_job'; +import { CSV_BOM_CHARS } from '../../../common/constants'; const delay = ms => new Promise(resolve => setTimeout(() => resolve(), ms)); @@ -374,6 +375,50 @@ describe('CSV Execute Job', function() { }); }); + describe('Byte order mark encoding', () => { + it('encodes CSVs with BOM', async () => { + configGetStub.withArgs('csv', 'useByteOrderMarkEncoding').returns(true); + callAsCurrentUserStub.onFirstCall().returns({ + hits: { + hits: [{ _source: { one: 'one', two: 'bar' } }], + }, + _scroll_id: 'scrollId', + }); + + const executeJob = await executeJobFactory(mockReportingPlugin, mockLogger); + const jobParams = { + headers: encryptedHeaders, + fields: ['one', 'two'], + conflictedTypesFields: [], + searchRequest: { index: null, body: null }, + }; + const { content } = await executeJob('job123', jobParams, cancellationToken); + + expect(content).toEqual(`${CSV_BOM_CHARS}one,two\none,bar\n`); + }); + + it('encodes CSVs without BOM', async () => { + configGetStub.withArgs('csv', 'useByteOrderMarkEncoding').returns(false); + callAsCurrentUserStub.onFirstCall().returns({ + hits: { + hits: [{ _source: { one: 'one', two: 'bar' } }], + }, + _scroll_id: 'scrollId', + }); + + const executeJob = await executeJobFactory(mockReportingPlugin, mockLogger); + const jobParams = { + headers: encryptedHeaders, + fields: ['one', 'two'], + conflictedTypesFields: [], + searchRequest: { index: null, body: null }, + }; + const { content } = await executeJob('job123', jobParams, cancellationToken); + + expect(content).toEqual('one,two\none,bar\n'); + }); + }); + describe('Elasticsearch call errors', function() { it('should reject Promise if search call errors out', async function() { callAsCurrentUserStub.rejects(new Error()); diff --git a/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.ts b/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.ts index 3a282eb0b2974..376a398da274f 100644 --- a/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.ts +++ b/x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import Hapi from 'hapi'; import { IUiSettingsClient, KibanaRequest } from '../../../../../../../src/core/server'; -import { CSV_JOB_TYPE } from '../../../common/constants'; +import { CSV_JOB_TYPE, CSV_BOM_CHARS } from '../../../common/constants'; import { ReportingCore } from '../../../server/core'; import { cryptoFactory } from '../../../server/lib'; import { getFieldFormats } from '../../../server/services'; @@ -121,6 +121,8 @@ export const executeJobFactory: ExecuteJobFactory