fix csv missing metadata fields issue and empty report on all(_source…

…) fields (#206)
opensearch-project · Nov 3, 2021 · 24cea4e · 24cea4e
1 parent c7b1c11
commit 24cea4e
Show file tree

Hide file tree

Showing 7 changed files with 182 additions and 39 deletions.
diff --git a/.github/workflows/dashboards-reports-test-and-build-workflow.yml b/.github/workflows/dashboards-reports-test-and-build-workflow.yml
@@ -5,7 +5,7 @@ on: [pull_request, push]
 env:
   PLUGIN_NAME: reportsDashboards
   ARTIFACT_NAME: reports-dashboards
-  OPENSEARCH_VERSION: '1.x'
+  OPENSEARCH_VERSION: '1.1'
   OPENSEARCH_PLUGIN_VERSION: 1.1.0.0
 
 jobs:

diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts
@@ -69,7 +69,8 @@ export const createReport = async (
   // @ts-ignore
   const timezone = request.query.timezone;
   // @ts-ignore
-  const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
+  const dateFormat =
+    request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
   const {
     basePath,
     serverInfo: { protocol, port, hostname },
@@ -97,7 +98,8 @@ export const createReport = async (
         report,
         opensearchClient,
         dateFormat,
-        isScheduledTask
+        isScheduledTask,
+        logger
       );
     } else {
       // report source can only be one of [saved search, visualization, dashboard, notebook]

diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts
@@ -27,6 +27,8 @@
 import 'regenerator-runtime/runtime';
 import { createSavedSearchReport } from '../savedSearchReportHelper';
 import { reportSchema } from '../../../model';
+import { mockLogger } from '../../../../test/__mocks__/loggerMock';
+import _ from 'lodash';
 
 /**
  * The mock and sample input for saved search export function.
@@ -54,14 +56,16 @@ const input = {
       configIds: [],
       title: 'title',
       textDescription: 'text description',
-      htmlDescription: 'html description'
+      htmlDescription: 'html description',
     },
     trigger: {
       trigger_type: 'On demand',
     },
   },
 };
 
+const mockDateFormat = 'date_hour_minute_second_fraction';
+
 /**
  * Max result window size in OpenSearch index settings.
  */
@@ -78,30 +82,45 @@ describe('test create saved search report', () => {
     const client = mockOpenSearchClient(hits);
     const { timeCreated, fileName } = await createSavedSearchReport(
       input,
-      client
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
     );
     expect(fileName).toContain(`test report table order_`);
   }, 20000);
 
   test('create report with expected file name extension', async () => {
     const csvReport = await createSavedSearchReport(
       input,
-      mockOpenSearchClient([])
+      mockOpenSearchClient([]),
+      mockDateFormat,
+      undefined,
+      mockLogger
     );
     expect(csvReport.fileName).toContain('.csv');
 
     input.report_definition.report_params.core_params.report_format = 'xlsx';
     const xlsxReport = await createSavedSearchReport(
       input,
-      mockOpenSearchClient([])
+      mockOpenSearchClient([]),
+      mockDateFormat,
+      undefined,
+      mockLogger
     );
     expect(xlsxReport.fileName).toContain('.xlsx');
   }, 20000);
 
   test('create report for empty data set', async () => {
     const hits: Array<{ _source: any }> = [];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
     expect(dataUrl).toEqual('');
   }, 20000);
 
@@ -114,7 +133,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -141,7 +166,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c11', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -171,7 +202,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male');
   }, 20000);
@@ -193,7 +230,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c10', customer_gender: 'Female' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -219,7 +262,13 @@ describe('test create saved search report', () => {
       hit({ category: 'c6', customer_gender: 'Female' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -239,7 +288,13 @@ describe('test create saved search report', () => {
       hit({ category: ',,c3', customer_gender: 'Male,,,' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -265,7 +320,13 @@ describe('test create saved search report', () => {
       hits,
       '"geoip.country_iso_code", "geoip.city_name", "geoip.location"'
     );
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' +
@@ -283,7 +344,13 @@ describe('test create saved search report', () => {
       hit({ category: ',,,@c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -307,7 +374,13 @@ describe('test create saved search report', () => {
       hit({ category: ',,,@c5', customer_gender: 'Male' }),
     ];
     const client = mockOpenSearchClient(hits);
-    const { dataUrl } = await createSavedSearchReport(input, client);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      undefined,
+      mockLogger
+    );
 
     expect(dataUrl).toEqual(
       'category,customer_gender\n' +
@@ -327,13 +400,54 @@ test('create report for data set contains null field value', async () => {
     hit({ category: 'c3', customer_gender: null }),
   ];
   const client = mockOpenSearchClient(hits);
-  const { dataUrl } = await createSavedSearchReport(input, client);
+  const { dataUrl } = await createSavedSearchReport(
+    input,
+    client,
+    mockDateFormat,
+    undefined,
+    mockLogger
+  );
 
   expect(dataUrl).toEqual(
     'category,customer_gender\n' + 'c1,Ma\n' + 'c2,le\n' + 'c3, '
   );
 }, 20000);
 
+test('create report for data set with metadata fields', async () => {
+  const metadataFields = { _index: 'nameofindex', _id: 'someid' };
+  let hits = [
+    hit({ category: 'c1', customer_gender: 'Male' }),
+    hit({ category: 'c2', customer_gender: 'Male' }),
+    hit({ category: 'c3', customer_gender: 'Male' }),
+    hit({ category: 'c4', customer_gender: 'Male' }),
+    hit({ category: 'c5', customer_gender: 'Male' }),
+  ];
+  hits.forEach((i) => {
+    _.merge(i, metadataFields);
+  });
+
+  const client = mockOpenSearchClient(
+    hits,
+    '"category", "customer_gender","_index","_id"'
+  );
+  const { dataUrl } = await createSavedSearchReport(
+    input,
+    client,
+    mockDateFormat,
+    undefined,
+    mockLogger
+  );
+
+  expect(dataUrl).toEqual(
+    'category,customer_gender,_index,_id\n' +
+      'c1,Male,nameofindex,someid\n' +
+      'c2,Male,nameofindex,someid\n' +
+      'c3,Male,nameofindex,someid\n' +
+      'c4,Male,nameofindex,someid\n' +
+      'c5,Male,nameofindex,someid'
+  );
+}, 20000);
+
 /**
  * Mock Elasticsearch client and return different mock objects based on endpoint and parameters.
  */

diff --git a/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts
@@ -28,18 +28,9 @@ import 'regenerator-runtime/runtime';
 import { createVisualReport } from '../visual_report/visualReportHelper';
 import { Logger } from '../../../../../../src/core/server';
 import { ReportParamsSchemaType, reportSchema } from '../../../model';
+import { mockLogger } from '../../../../test/__mocks__/loggerMock';
 
-const mockLogger: Logger = {
-  info: jest.fn(),
-  trace: jest.fn(),
-  warn: jest.fn(),
-  debug: jest.fn(),
-  error: jest.fn(),
-  fatal: jest.fn(),
-  log: jest.fn(),
-  get: jest.fn(),
-};
-
+const mockHeader = { mockKey: 'mockValue' };
 const input = {
   query_url: '/app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d',
   time_from: 1343576635300,
@@ -84,7 +75,8 @@ describe('test create visual report', () => {
     const { dataUrl, fileName } = await createVisualReport(
       reportParams as ReportParamsSchemaType,
       mockHtmlPath,
-      mockLogger
+      mockLogger,
+      mockHeader
     );
     expect(fileName).toContain(`${reportParams.report_name}`);
     expect(fileName).toContain('.png');
@@ -99,7 +91,8 @@ describe('test create visual report', () => {
     const { dataUrl, fileName } = await createVisualReport(
       reportParams as ReportParamsSchemaType,
       mockHtmlPath,
-      mockLogger
+      mockLogger,
+      mockHeader
     );
     expect(fileName).toContain(`${reportParams.report_name}`);
     expect(fileName).toContain('.pdf');

diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts
@@ -49,14 +49,17 @@ export var metaData = {
 // Get the selected columns by the user.
 export const getSelectedFields = async (columns) => {
   const selectedFields = [];
+  let fields_exist = false;
   for (let column of columns) {
     if (column !== '_source') {
-      metaData.fields_exist = true;
+      fields_exist = true;
       selectedFields.push(column);
     } else {
+      fields_exist = false;
       selectedFields.push('_source');
     }
   }
+  metaData.fields_exist = fields_exist;
   metaData.selectedFields = selectedFields;
 };
 
@@ -191,7 +194,7 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string)
       }
       delete data['fields'];
       if (report._source.fields_exist === true) {
-        let result = traverse(data._source, report._source.selectedFields);
+        let result = traverse(data, report._source.selectedFields);
         hits.push(params.excel ? sanitize(result) : result);
       } else {
         hits.push(params.excel ? sanitize(data) : data);
@@ -229,7 +232,7 @@ function flattenHits(hits, result = {}, prefix = '') {
     ) {
       flattenHits(value, result, prefix + key + '.');
     } else {
-      result[prefix + key] = value;
+      result[prefix.replace(/^_source\./, '') + key] = value;
     }
   }
   return result;