Update import api to have data source id to allow import saved objects from uploading files to have data source

CHANGELOG.md

@@ -344,6 +344,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Dashboard De-Angular] Add more unit tests for utils folder ([#4641](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4641))
 - [Dashboard De-Angular] Add unit tests for dashboard_listing and dashboard_top_nav ([#4640](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4640))
 - Optimize `augment-vis` saved obj searching by adding arg to saved obj client ([#4595](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4595))
+- Change SavedObjects' Import API to allow selecting a data source when uploading files ([#5777](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5777))
 
 ### 🐛 Bug Fixes
 

src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts

@@ -0,0 +1,166 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { mockUuidv4 } from './__mocks__';
+import { SavedObjectReference, SavedObjectsImportRetry } from 'opensearch-dashboards/public';
+import { SavedObject } from '../types';
+import { SavedObjectsErrorHelpers } from '..';
+import {
+  checkConflictsForDataSource,
+  ConflictsForDataSourceParams,
+} from './check_conflict_for_data_source';
+
+type SavedObjectType = SavedObject<{ title?: string }>;
+
+/**
+ * Function to create a realistic-looking import object given a type and ID
+ */
+const createObject = (type: string, id: string): SavedObjectType => ({
+  type,
+  id,
+  attributes: { title: 'some-title' },
+  references: (Symbol() as unknown) as SavedObjectReference[],
+});
+
+const getResultMock = {
+  conflict: (type: string, id: string) => {
+    const error = SavedObjectsErrorHelpers.createConflictError(type, id).output.payload;
+    return { type, id, error };
+  },
+  unresolvableConflict: (type: string, id: string) => {
+    const conflictMock = getResultMock.conflict(type, id);
+    const metadata = { isNotOverwritable: true };
+    return { ...conflictMock, error: { ...conflictMock.error, metadata } };
+  },
+  invalidType: (type: string, id: string) => {
+    const error = SavedObjectsErrorHelpers.createUnsupportedTypeError(type).output.payload;
+    return { type, id, error };
+  },
+};
+
+/**
+ * Create a variety of different objects to exercise different import / result scenarios
+ */
+const obj1 = createObject('type-1', 'id-1'); // -> success
+const obj2 = createObject('type-2', 'id-2'); // -> conflict
+const obj3 = createObject('type-3', 'id-3'); // -> unresolvable conflict
+const obj4 = createObject('type-4', 'id-4'); // -> invalid type
+const dataSourceObj = createObject('data-source', 'data-source-id-1'); // -> data-source type, no need to add in the filteredObjects
+const objects = [obj1, obj2, obj3, obj4];
+const dataSourceObj1 = createObject('type-1', 'ds_id-1'); // -> object with data source id
+const dataSourceObj2 = createObject('type-2', 'ds_id-2'); // -> object with data source id
+const objectsWithDataSource = [dataSourceObj, dataSourceObj1, dataSourceObj2];
+const dataSourceObj1Error = getResultMock.conflict(dataSourceObj1.type, dataSourceObj1.id);
+
+describe('#checkConflictsForDataSource', () => {
+  const setupParams = (partial: {
+    objects: SavedObjectType[];
+    ignoreRegularConflicts?: boolean;
+    retries?: SavedObjectsImportRetry[];
+    createNewCopies?: boolean;
+    dataSourceId?: string;
+  }): ConflictsForDataSourceParams => {
+    return { ...partial };
+  };
+
+  beforeEach(() => {
+    mockUuidv4.mockReset();
+    mockUuidv4.mockReturnValueOnce(`new-object-id`);
+  });
+
+  it('exits early if there are no objects to check', async () => {
+    const params = setupParams({ objects: [] });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('returns original objects result when there is no data source id', async () => {
+    const params = setupParams({ objects });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [...objects],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('return obj if it is not data source obj and there is no conflict of the data source id', async () => {
+    const params = setupParams({ objects: objectsWithDataSource, dataSourceId: 'ds' });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [dataSourceObj1, dataSourceObj2],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('can resolve the data source id conflict when the ds it not match when ignoreRegularConflicts=true', async () => {
+    const params = setupParams({
+      objects: objectsWithDataSource,
+      ignoreRegularConflicts: true,
+      dataSourceId: 'currentDsId',
+    });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+
+    expect(checkConflictsForDataSourceResult).toEqual(
+      expect.objectContaining({
+        filteredObjects: [
+          {
+            ...dataSourceObj1,
+            id: 'currentDsId_id-1',
+          },
+          {
+            ...dataSourceObj2,
+            id: 'currentDsId_id-2',
+          },
+        ],
+        errors: [],
+        importIdMap: new Map([
+          [
+            `${dataSourceObj1.type}:${dataSourceObj1.id}`,
+            { id: 'currentDsId_id-1', omitOriginId: true },
+          ],
+          [
+            `${dataSourceObj2.type}:${dataSourceObj2.id}`,
+            { id: 'currentDsId_id-2', omitOriginId: true },
+          ],
+        ]),
+        pendingOverwrites: new Set([
+          `${dataSourceObj1.type}:${dataSourceObj1.id}`,
+          `${dataSourceObj2.type}:${dataSourceObj2.id}`,
+        ]),
+      })
+    );
+  });
+
+  it('can push error when do not override with data source conflict', async () => {
+    const params = setupParams({
+      objects: [dataSourceObj1],
+      ignoreRegularConflicts: false,
+      dataSourceId: 'currentDs',
+    });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [],
+      errors: [
+        {
+          ...dataSourceObj1Error,
+          title: dataSourceObj1.attributes.title,
+          meta: { title: dataSourceObj1.attributes.title },
+          error: { type: 'conflict' },
+        },
+      ],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+});

src/core/server/saved_objects/import/check_conflict_for_data_source.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SavedObject, SavedObjectsImportError, SavedObjectsImportRetry } from '../types';
+
+export interface ConflictsForDataSourceParams {
+  objects: Array<SavedObject<{ title?: string }>>;
+  ignoreRegularConflicts?: boolean;
+  retries?: SavedObjectsImportRetry[];
+  dataSourceId?: string;
+}
+
+interface ImportIdMapEntry {
+  id?: string;
+  omitOriginId?: boolean;
+}
+
+/**
+ * function to check the conflict when multiple data sources are enabled.
+ * the purpose of this function is to check the conflict of the imported saved objects and data source
+ * @param objects, this the array of saved objects to be verified whether contains the data source conflict
+ * @param ignoreRegularConflicts whether to override
+ * @param retries import operations list
+ * @param dataSourceId the id to identify the data source
+ * @returns {filteredObjects, errors, importIdMap, pendingOverwrites }
+ */
+export async function checkConflictsForDataSource({
+  objects,
+  ignoreRegularConflicts,
+  retries = [],
+  dataSourceId,
+}: ConflictsForDataSourceParams) {
+  const filteredObjects: Array<SavedObject<{ title?: string }>> = [];
+  const errors: SavedObjectsImportError[] = [];
+  const importIdMap = new Map<string, ImportIdMapEntry>();
+  const pendingOverwrites = new Set<string>();
+
+  // exit early if there are no objects to check
+  if (objects.length === 0) {
+    return { filteredObjects, errors, importIdMap, pendingOverwrites };
+  }
+  const retryMap = retries.reduce(
+    (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur),
+    new Map<string, SavedObjectsImportRetry>()
+  );
+  objects.forEach((object) => {
+    const {
+      type,
+      id,
+      attributes: { title },
+    } = object;
+    const { destinationId } = retryMap.get(`${type}:${id}`) || {};
+
+    if (!object.type.includes('data-source')) {
+      // check the previous data source existed or not
+      // by extract it from the id
+      // e.g. e0c9e490-bdd7-11ee-b216-d78a57002330_ff959d40-b880-11e8-a6d9-e546fe2bba5f
+      // e0c9e490-bdd7-11ee-b216-d78a57002330 is the data source id
+      // for saved object data source itself, e0c9e490-bdd7-11ee-b216-d78a57002330 return undefined
+      const parts = id.split('_'); // this is the array to host the split results of the id
+      const previoudDataSourceId = parts.length > 1 ? parts[0] : undefined;
+      /**
+       * for import saved object from osd exported
+       * when the imported saved objects with the different dataSourceId comparing to the current dataSourceId
+       */
+      if (!previoudDataSourceId || previoudDataSourceId === dataSourceId) {
+        filteredObjects.push(object);
+      } else if (previoudDataSourceId && previoudDataSourceId !== dataSourceId) {
+        if (ignoreRegularConflicts) {
+          /**
+           * use old key and new value in the importIdMap
+           */
+          const omitOriginId = ignoreRegularConflicts;
+          const rawId = parts[1];
+          importIdMap.set(`${type}:${id}`, { id: `${dataSourceId}_${rawId}`, omitOriginId });
+          pendingOverwrites.add(`${type}:${id}`);
+          filteredObjects.push({ ...object, id: `${dataSourceId}_${rawId}` });
+        } else {
+          // not override
+          // push error
+          const error = { type: 'conflict' as 'conflict', ...(destinationId && { destinationId }) };
+          errors.push({ type, id, title, meta: { title }, error });
+        }
+      }
+    }
+  });
+
+  return { filteredObjects, errors, importIdMap, pendingOverwrites };
+}

src/core/server/saved_objects/import/check_conflicts.ts

@@ -66,7 +66,6 @@ export async function checkConflicts({
   if (objects.length === 0) {
     return { filteredObjects, errors, importIdMap, pendingOverwrites };
   }
-
   const retryMap = retries.reduce(
     (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur),
     new Map<string, SavedObjectsImportRetry>()

src/core/server/saved_objects/import/check_origin_conflicts.ts

@@ -45,6 +45,7 @@ interface CheckOriginConflictsParams {
   namespace?: string;
   ignoreRegularConflicts?: boolean;
   importIdMap: Map<string, unknown>;
+  dataSourceId?: string;
 }
 
 type CheckOriginConflictParams = Omit<CheckOriginConflictsParams, 'objects'> & {

src/core/server/saved_objects/import/collect_saved_objects.ts

@@ -46,13 +46,15 @@ interface CollectSavedObjectsOptions {
   objectLimit: number;
   filter?: (obj: SavedObject) => boolean;
   supportedTypes: string[];
+  dataSourceId?: string;
 }
 
 export async function collectSavedObjects({
   readStream,
   objectLimit,
   filter,
   supportedTypes,
+  dataSourceId,
 }: CollectSavedObjectsOptions) {
   const errors: SavedObjectsImportError[] = [];
   const entries: Array<{ type: string; id: string }> = [];