From 83bf1b4cd467ca6877f4d7b19bd0a3fb69215be6 Mon Sep 17 00:00:00 2001 From: Joshua Date: Tue, 25 May 2021 14:54:52 -0700 Subject: [PATCH] Fix csv parsing function (#53) * Fix csv parsing function Signed-off-by: Joshua Li * Flatten nested results for csv Signed-off-by: Joshua Li * Add test case for nested fields Signed-off-by: Joshua Li --- .../__tests__/savedSearchReportHelper.test.ts | 41 ++++++++++++++---- .../server/routes/utils/dataReportHelpers.ts | 42 ++++++++++++------- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 3655a006..f1ca9b8a 100644 --- a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -249,6 +249,31 @@ describe('test create saved search report', () => { ); }, 20000); + test('create report for data set with nested fields', async () => { + const hits = [ + hit({ + 'geoip.country_iso_code': 'GB', + 'geoip.location': { lon: -0.1, lat: 51.5 }, + }), + hit({ + 'geoip.country_iso_code': 'US', + 'geoip.city_name': 'New York', + 'geoip.location': { lon: -74, lat: 40.8 }, + }), + ]; + const client = mockOpenSearchClient( + hits, + '"geoip.country_iso_code", "geoip.city_name", "geoip.location"' + ); + const { dataUrl } = await createSavedSearchReport(input, client); + + expect(dataUrl).toEqual( + 'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' + + 'GB,-0.1,51.5, \n' + + 'US,-74,40.8,New York' + ); + }, 20000); + test('create report by sanitizing data set for Excel', async () => { const hits = [ hit({ category: 'c1', customer_gender: '=Male' }), @@ -312,7 +337,10 @@ test('create report for data set contains null field value', async () => { /** * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. */ -function mockOpenSearchClient(mockHits: Array<{ _source: any }>) { +function mockOpenSearchClient( + mockHits: Array<{ _source: any }>, + columns = '"category", "customer_gender"' +) { let call = 0; const client = jest.fn(); client.callAsInternalUser = jest @@ -323,7 +351,7 @@ function mockOpenSearchClient(mockHits: Array<{ _source: any }>) { return { _source: params.id.startsWith('index-pattern:') ? mockIndexPattern() - : mockSavedSearch(), + : mockSavedSearch(columns), }; case 'indices.getSettings': return mockIndexSettings(); @@ -357,9 +385,9 @@ function mockOpenSearchClient(mockHits: Array<{ _source: any }>) { } /** - * Mock a saved search for opensearch_dashboards_sample_data_ecommerce with 2 selected fields: category and customer_gender. + * Mock a saved search for opensearch_dashboards_sample_data_ecommerce with 2 default selected fields: category and customer_gender. */ -function mockSavedSearch() { +function mockSavedSearch(columns = '"category", "customer_gender"') { return JSON.parse(` { "type": "search", @@ -368,10 +396,7 @@ function mockSavedSearch() { "title": "Show category and gender", "description": "", "hits": 0, - "columns": [ - "category", - "customer_gender" - ], + "columns": [ ${columns} ], "sort": [], "version": 1, "opensearchDashboardsSavedObjectMeta": { diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index 3b56f6ed..3840121b 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -29,6 +29,7 @@ import { DATA_REPORT_CONFIG } from './constants'; import esb from 'elastic-builder'; import moment from 'moment'; import converter from 'json-2-csv'; +import _ from 'lodash'; export var metaData = { saved_search_id: null, @@ -181,7 +182,7 @@ export const getOpenSearchData = (arrayHits, report, params) => { } delete data['fields']; if (report._source.fields_exist === true) { - let result = traverse(data, report._source.selectedFields); + let result = traverse(data._source, report._source.selectedFields); hits.push(params.excel ? sanitize(result) : result); } else { hits.push(params.excel ? sanitize(data) : data); @@ -208,26 +209,39 @@ export const convertToCSV = async (dataset) => { return convertedData; }; -//Return only the selected fields -function traverse(data, keys, result = {}) { - for (let k of Object.keys(data)) { - if (keys.includes(k)) { - result = Object.assign({}, result, { - [k]: data[k], - }); - continue; - } +function flattenHits(hits, result = {}, prefix = '') { + for (const [key, value] of Object.entries(hits)) { + if (!hits.hasOwnProperty(key)) continue; if ( - data[k] && - typeof data[k] === 'object' && - Object.keys(data[k]).length > 0 + value != null && + typeof value === 'object' && + !Array.isArray(value) && + Object.keys(value).length > 0 ) { - result = traverse(data[k], keys, result); + flattenHits(value, result, prefix + key + '.'); + } else { + result[prefix + key] = value; } } return result; } +//Return only the selected fields +function traverse(data, keys, result = {}) { + data = flattenHits(data); + const sourceKeys = Object.keys(data); + keys.forEach((key) => { + const value = _.get(data, key, undefined); + if (value !== undefined) result[key] = value; + else { + Object.keys(data) + .filter((sourceKey) => sourceKey.startsWith(key + '.')) + .forEach((sourceKey) => (result[sourceKey] = data[sourceKey])); + } + }); + return result; +} + /** * Escape special characters if field value prefixed with. * This is intend to avoid CSV injection in Microsoft Excel.