Use advanced settings for csv separator and visual report timezone

dashboards-reports/public/components/context_menu/context_menu.js

@@ -105,9 +105,9 @@ const generateInContextReport = async (
   };
 
   fetch(
-    `../api/reporting/generateReport?timezone=${
-      Intl.DateTimeFormat().resolvedOptions().timeZone
-    }&dateFormat=${uiSettingsService.get('dateFormat')}`,
+    `../api/reporting/generateReport?${new URLSearchParams(
+      uiSettingsService.getSearchParams()
+    )}`,
     {
       headers: {
         'Content-Type': 'application/json',

dashboards-reports/public/components/main/main_utils.tsx

@@ -181,10 +181,7 @@ export const generateReportFromDefinitionId = async (
       headers: {
         'Content-Type': 'application/json',
       },
-      query: {
-        timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
-        dateFormat: uiSettingsService.get('dateFormat'),
-      },
+      query: uiSettingsService.getSearchParams(),
     })
     .then(async (response: any) => {
       // for emailing a report, this API response doesn't have response body
@@ -217,10 +214,7 @@ export const generateReportById = async (
 ) => {
   await httpClient
     .get(`../api/reporting/generateReport/${reportId}`, {
-      query: {
-        timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
-        dateFormat: uiSettingsService.get('dateFormat'),
-      },
+      query: uiSettingsService.getSearchParams(),
     })
     .then(async (response) => {
       //TODO: duplicate code, extract to be a function that can reuse. e.g. handleResponse(response)

dashboards-reports/public/components/utils/settings_service.ts

@@ -20,4 +20,18 @@ export const uiSettingsService = {
   get: (key: string, defaultOverride?: any) => {
     return uiSettings?.get(key, defaultOverride) || '';
   },
+  getSearchParams: function () {
+    const rawTimeZone = this.get('dateFormat:tz');
+    const timezone =
+      !rawTimeZone || rawTimeZone === 'Browser'
+        ? Intl.DateTimeFormat().resolvedOptions().timeZone
+        : rawTimeZone;
+    const dateFormat = this.get('dateFormat');
+    const csvSeparator = this.get('csv:separator');
+    return {
+      timezone,
+      dateFormat,
+      csvSeparator,
+    };
+  },
 };

dashboards-reports/server/routes/lib/createReport.ts

@@ -71,6 +71,8 @@ export const createReport = async (
   // @ts-ignore
   const dateFormat =
     request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
+  // @ts-ignore
+  const csvSeparator = request.query.csvSeparator || ',';
   const {
     basePath,
     serverInfo: { protocol, port, hostname },
@@ -98,6 +100,7 @@ export const createReport = async (
         report,
         opensearchClient,
         dateFormat,
+        csvSeparator,
         isScheduledTask,
         logger
       );

dashboards-reports/server/routes/report.ts

@@ -58,6 +58,7 @@ export default function (router: IRouter, accessInfo: AccessInfoType) {
         query: schema.object({
           timezone: schema.maybe(schema.string()),
           dateFormat: schema.maybe(schema.string()),
+          csvSeparator: schema.maybe(schema.string()),
         }),
       },
     },
@@ -122,7 +123,8 @@ export default function (router: IRouter, accessInfo: AccessInfoType) {
         }),
         query: schema.object({
           timezone: schema.string(),
-          dateFormat: schema.maybe(schema.string()),
+          dateFormat: schema.string(),
+          csvSeparator: schema.string(),
         }),
       },
     },
@@ -188,7 +190,8 @@ export default function (router: IRouter, accessInfo: AccessInfoType) {
         }),
         query: schema.object({
           timezone: schema.string(),
-          dateFormat: schema.maybe(schema.string()),
+          dateFormat: schema.string(),
+          csvSeparator: schema.string(),
         }),
       },
     },

dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts

@@ -64,7 +64,7 @@ const input = {
   },
 };
 
-const mockDateFormat = 'date_hour_minute_second_fraction';
+const mockDateFormat = 'MM/DD/YYYY h:mm:ss.SSS a';
 
 /**
  * Max result window size in OpenSearch index settings.
@@ -84,6 +84,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -95,6 +96,7 @@ describe('test create saved search report', () => {
       input,
       mockOpenSearchClient([]),
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -105,6 +107,7 @@ describe('test create saved search report', () => {
       input,
       mockOpenSearchClient([]),
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -118,6 +121,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -137,6 +141,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -170,6 +175,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -206,6 +212,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -234,6 +241,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -266,6 +274,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -292,6 +301,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -304,6 +314,30 @@ describe('test create saved search report', () => {
     );
   }, 20000);
 
+  test('create report for data set with comma and custom separator', async () => {
+    const hits = [
+      hit({ category: ',c1', customer_gender: 'Ma,le' }),
+      hit({ category: 'c2,', customer_gender: 'M,ale' }),
+      hit({ category: ',,c3', customer_gender: 'Male,,,' }),
+    ];
+    const client = mockOpenSearchClient(hits);
+    const { dataUrl } = await createSavedSearchReport(
+      input,
+      client,
+      mockDateFormat,
+      '|',
+      undefined,
+      mockLogger
+    );
+
+    expect(dataUrl).toEqual(
+      'category|customer_gender\n' +
+        ',c1|Ma,le\n' +
+        'c2,|M,ale\n' +
+        ',,c3|Male,,,'
+    );
+  }, 20000);
+
   test('create report for data set with nested fields', async () => {
     const hits = [
       hit({
@@ -324,6 +358,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -348,6 +383,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -378,6 +414,7 @@ describe('test create saved search report', () => {
       input,
       client,
       mockDateFormat,
+      ',',
       undefined,
       mockLogger
     );
@@ -404,6 +441,7 @@ test('create report for data set contains null field value', async () => {
     input,
     client,
     mockDateFormat,
+    ',',
     undefined,
     mockLogger
   );
@@ -434,6 +472,7 @@ test('create report for data set with metadata fields', async () => {
     input,
     client,
     mockDateFormat,
+    ',',
     undefined,
     mockLogger
   );

dashboards-reports/server/routes/utils/dataReportHelpers.ts

@@ -163,10 +163,10 @@ export const getOpenSearchData = (
 };
 
 //Convert the data to Csv format
-export const convertToCSV = async (dataset) => {
+export const convertToCSV = async (dataset, csvSeparator) => {
   let convertedData: any = [];
   const options = {
-    delimiter: { field: ',', eol: '\n' },
+    delimiter: { field: csvSeparator, eol: '\n' },
     emptyFieldValue: ' ',
   };
   await converter.json2csvAsync(dataset[0], options).then((csv) => {

dashboards-reports/server/routes/utils/savedSearchReportHelper.ts

@@ -50,6 +50,7 @@ export async function createSavedSearchReport(
   report: any,
   client: ILegacyClusterClient | ILegacyScopedClusterClient,
   dateFormat: string,
+  csvSeparator: string,
   isScheduledTask: boolean = true,
   logger: Logger
 ): Promise<CreateReportResultType> {
@@ -62,6 +63,7 @@ export async function createSavedSearchReport(
     client,
     params.core_params,
     dateFormat,
+    csvSeparator,
     isScheduledTask,
     logger
   );
@@ -148,6 +150,7 @@ async function generateReportData(
   client: ILegacyClusterClient | ILegacyScopedClusterClient,
   params: any,
   dateFormat: string,
+  csvSeparator: string,
   isScheduledTask: boolean,
   logger: Logger
 ) {
@@ -277,6 +280,6 @@ async function generateReportData(
   async function convertOpenSearchDataToCsv() {
     const dataset: any = [];
     dataset.push(getOpenSearchData(arrayHits, report, params, dateFormat));
-    return await convertToCSV(dataset);
+    return await convertToCSV(dataset, csvSeparator);
   }
 }