[security solutions][lists] Adds end to end tests (#74473) (#74768)

## Summary

Adds initial set of end to end tests for lists

You can run all of these with the command from kibana root:

```ts
node scripts/functional_tests --config x-pack/test/lists_api_integration/security_and_spaces/config.ts
```

Fixes a few minor bugs found such as...
* Validation for importing lists was not checking if the indexes were created first
* Some wording for the error messages had duplicate words within them

### Checklist

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

x-pack/plugins/lists/common/constants.mock.ts

@@ -9,6 +9,7 @@ import { EntriesArray } from './schemas/types';
 export const DATE_NOW = '2020-04-20T15:25:31.830Z';
 export const OLD_DATE_RELATIVE_TO_DATE_NOW = '2020-04-19T15:25:31.830Z';
 export const USER = 'some user';
+export const ELASTIC_USER = 'elastic';
 export const LIST_INDEX = '.lists';
 export const LIST_ITEM_INDEX = '.items';
 export const NAME = 'some name';

x-pack/plugins/lists/common/schemas/request/create_list_item_schema.mock.ts

@@ -14,3 +14,20 @@ export const getCreateListItemSchemaMock = (): CreateListItemSchema => ({
   meta: META,
   value: VALUE,
 });
+
+/**
+ * Useful for end to end testing
+ */
+export const getCreateMinimalListItemSchemaMock = (): CreateListItemSchema => ({
+  id: LIST_ITEM_ID,
+  list_id: LIST_ID,
+  value: VALUE,
+});
+
+/**
+ * Useful for end to end testing
+ */
+export const getCreateMinimalListItemSchemaMockWithoutId = (): CreateListItemSchema => ({
+  list_id: LIST_ID,
+  value: VALUE,
+});

x-pack/plugins/lists/common/schemas/request/create_list_schema.mock.ts

@@ -18,3 +18,22 @@ export const getCreateListSchemaMock = (): CreateListSchema => ({
   type: TYPE,
   version: VERSION,
 });
+
+/**
+ * Useful for end to end tests and other mechanisms which want to fill in the values
+ */
+export const getCreateMinimalListSchemaMock = (): CreateListSchema => ({
+  description: DESCRIPTION,
+  id: LIST_ID,
+  name: NAME,
+  type: TYPE,
+});
+
+/**
+ * Useful for end to end tests and other mechanisms which want to fill in the values
+ */
+export const getCreateMinimalListSchemaMockWithoutId = (): CreateListSchema => ({
+  description: DESCRIPTION,
+  name: NAME,
+  type: TYPE,
+});

x-pack/plugins/lists/common/schemas/request/import_list_item_schema.mock.ts

@@ -9,3 +9,12 @@ import { ImportListItemSchema } from './import_list_item_schema';
 export const getImportListItemSchemaMock = (): ImportListItemSchema => ({
   file: {},
 });
+
+/**
+ * This is useful for end to end tests, it will return a buffer given a string array
+ * of things to import.
+ * @param input Array of strings of things to import
+ */
+export const getImportListItemAsBuffer = (input: string[]): Buffer => {
+  return Buffer.from(input.join('\r\n'));
+};

x-pack/plugins/lists/common/schemas/request/update_list_item_schema.mock.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ID, META, VALUE } from '../../constants.mock';
+import { ID, LIST_ITEM_ID, META, VALUE } from '../../constants.mock';
 
 import { UpdateListItemSchema } from './update_list_item_schema';
 
@@ -13,3 +13,11 @@ export const getUpdateListItemSchemaMock = (): UpdateListItemSchema => ({
   meta: META,
   value: VALUE,
 });
+
+/**
+ * Useful for end to end testing
+ */
+export const getUpdateMinimalListItemSchemaMock = (): UpdateListItemSchema => ({
+  id: LIST_ITEM_ID,
+  value: VALUE,
+});

x-pack/plugins/lists/common/schemas/request/update_list_schema.mock.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DESCRIPTION, LIST_ID, META, NAME, _VERSION } from '../../constants.mock';
+
+import { UpdateListSchema } from './update_list_schema';
+
+export const getUpdateListSchemaMock = (): UpdateListSchema => ({
+  _version: _VERSION,
+  description: DESCRIPTION,
+  id: LIST_ID,
+  meta: META,
+  name: NAME,
+});
+
+/**
+ * Useful for end to end tests and other mechanisms which want to fill in the values
+ * after doing a get of the structure.
+ */
+export const getUpdateMinimalListSchemaMock = (): UpdateListSchema => ({
+  description: DESCRIPTION,
+  id: LIST_ID,
+  name: NAME,
+});

x-pack/plugins/lists/common/schemas/request/update_list_schema.test.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { left } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+
+import { exactCheck, foldLeftRight, getPaths } from '../../shared_imports';
+
+import { UpdateListSchema, updateListSchema } from './update_list_schema';
+import { getUpdateListSchemaMock } from './update_list_schema.mock';
+
+describe('update_list_schema', () => {
+  test('it should validate a typical list request', () => {
+    const payload = getUpdateListSchemaMock();
+    const decoded = updateListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should accept an undefined for "meta" but strip it out', () => {
+    const payload = getUpdateListSchemaMock();
+    const outputPayload = getUpdateListSchemaMock();
+    delete payload.meta;
+    const decoded = updateListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    delete outputPayload.meta;
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(outputPayload);
+  });
+
+  test('it should not allow an extra key to be sent in', () => {
+    const payload: UpdateListSchema & {
+      extraKey?: string;
+    } = getUpdateListSchemaMock();
+    payload.extraKey = 'some new value';
+    const decoded = updateListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual(['invalid keys "extraKey"']);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts

@@ -7,6 +7,7 @@
 import { ListItemSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
+  ELASTIC_USER,
   LIST_ID,
   LIST_ITEM_ID,
   META,
@@ -31,3 +32,15 @@ export const getListItemResponseMock = (): ListItemSchema => ({
   updated_by: USER,
   value: VALUE,
 });
+
+/**
+ * This is useful for end to end tests where we remove the auto generated parts for comparisons
+ * such as created_at, updated_at, and id.
+ */
+export const getListItemResponseMockWithoutAutoGeneratedValues = (): Partial<ListItemSchema> => ({
+  created_by: ELASTIC_USER,
+  list_id: LIST_ID,
+  type: TYPE,
+  updated_by: ELASTIC_USER,
+  value: VALUE,
+});

x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts

@@ -8,6 +8,7 @@ import { ListSchema } from '../../../common/schemas';
 import {
   DATE_NOW,
   DESCRIPTION,
+  ELASTIC_USER,
   IMMUTABLE,
   LIST_ID,
   META,
@@ -35,3 +36,17 @@ export const getListResponseMock = (): ListSchema => ({
   updated_by: USER,
   version: VERSION,
 });
+
+/**
+ * This is useful for end to end tests where we remove the auto generated parts for comparisons
+ * such as created_at, updated_at, and id.
+ */
+export const getListResponseMockWithoutAutoGeneratedValues = (): Partial<ListSchema> => ({
+  created_by: ELASTIC_USER,
+  description: DESCRIPTION,
+  immutable: IMMUTABLE,
+  name: NAME,
+  type: TYPE,
+  updated_by: ELASTIC_USER,
+  version: VERSION,
+});

x-pack/plugins/lists/server/routes/create_list_item_route.ts

@@ -36,6 +36,15 @@ export const createListItemRoute = (router: IRouter): void => {
             statusCode: 404,
           });
         } else {
+          if (id != null) {
+            const listItem = await lists.getListItem({ id });
+            if (listItem != null) {
+              return siemResponse.error({
+                body: `list item id: "${id}" already exists`,
+                statusCode: 409,
+              });
+            }
+          }
           const createdListItem = await lists.createListItem({
             deserializer: list.deserializer,
             id,

x-pack/plugins/lists/server/routes/delete_list_item_route.ts

@@ -33,7 +33,7 @@ export const deleteListItemRoute = (router: IRouter): void => {
           const deleted = await lists.deleteListItem({ id });
           if (deleted == null) {
             return siemResponse.error({
-              body: `list item with id: "${id}" item not found`,
+              body: `list item with id: "${id}" not found`,
               statusCode: 404,
             });
           } else {

x-pack/plugins/lists/server/routes/import_list_item_route.ts

@@ -41,6 +41,13 @@ export const importListItemRoute = (router: IRouter, config: ConfigType): void =
         const stream = createStreamFromBuffer(request.body);
         const { deserializer, list_id: listId, serializer, type } = request.query;
         const lists = getListClient(context);
+        const listExists = await lists.getListIndexExists();
+        if (!listExists) {
+          return siemResponse.error({
+            body: `To import a list item, the index must exist first. Index "${lists.getListIndex()}" does not exist`,
+            statusCode: 400,
+          });
+        }
         if (listId != null) {
           const list = await lists.getList({ id: listId });
           if (list == null) {

x-pack/plugins/lists/server/routes/patch_list_route.ts

@@ -32,7 +32,7 @@ export const patchListRoute = (router: IRouter): void => {
         const list = await lists.updateList({ _version, description, id, meta, name, version });
         if (list == null) {
           return siemResponse.error({
-            body: `list id: "${id}" found found`,
+            body: `list id: "${id}" not found`,
             statusCode: 404,
           });
         } else {

x-pack/plugins/lists/server/routes/update_exception_list_route.ts

@@ -69,7 +69,7 @@ export const updateExceptionListRoute = (router: IRouter): void => {
           });
           if (list == null) {
             return siemResponse.error({
-              body: `exception list id: "${id}" found found`,
+              body: `exception list id: "${id}" not found`,
               statusCode: 404,
             });
           } else {

x-pack/plugins/lists/server/routes/update_list_route.ts

@@ -32,7 +32,7 @@ export const updateListRoute = (router: IRouter): void => {
         const list = await lists.updateList({ _version, description, id, meta, name, version });
         if (list == null) {
           return siemResponse.error({
-            body: `list id: "${id}" found found`,
+            body: `list id: "${id}" not found`,
             statusCode: 404,
           });
         } else {

x-pack/test/detection_engine_api_integration/common/config.ts

@@ -73,7 +73,6 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
           ])}`,
           `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`,
           '--xpack.eventLog.logEntries=true',
-          '--xpack.lists.enabled=true',
           ...disabledPlugins.map((key) => `--xpack.${key}.enabled=false`),
           `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'alerts')}`,
           `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'actions')}`,

x-pack/test/lists_api_integration/common/config.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import path from 'path';
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+import { services } from './services';
+
+interface CreateTestConfigOptions {
+  license: string;
+  disabledPlugins?: string[];
+  ssl?: boolean;
+}
+
+export function createTestConfig(name: string, options: CreateTestConfigOptions) {
+  const { license = 'trial', disabledPlugins = [], ssl = false } = options;
+
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const xPackApiIntegrationTestsConfig = await readConfigFile(
+      require.resolve('../../api_integration/config.ts')
+    );
+    const servers = {
+      ...xPackApiIntegrationTestsConfig.get('servers'),
+      elasticsearch: {
+        ...xPackApiIntegrationTestsConfig.get('servers.elasticsearch'),
+        protocol: ssl ? 'https' : 'http',
+      },
+    };
+
+    return {
+      testFiles: [require.resolve(`../${name}/tests/`)],
+      servers,
+      services,
+      junit: {
+        reportName: 'X-Pack Lists Integration Tests',
+      },
+      esArchiver: xPackApiIntegrationTestsConfig.get('esArchiver'),
+      esTestCluster: {
+        ...xPackApiIntegrationTestsConfig.get('esTestCluster'),
+        license,
+        ssl,
+        serverArgs: [
+          `xpack.license.self_generated.type=${license}`,
+          `xpack.security.enabled=${!disabledPlugins.includes('security')}`,
+        ],
+      },
+      kbnTestServer: {
+        ...xPackApiIntegrationTestsConfig.get('kbnTestServer'),
+        serverArgs: [
+          ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'),
+          ...disabledPlugins.map((key) => `--xpack.${key}.enabled=false`),
+          `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'alerts')}`,
+          `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'actions')}`,
+          `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'task_manager')}`,
+          `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'aad')}`,
+          ...(ssl
+            ? [
+                `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`,
+                `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+              ]
+            : []),
+        ],
+      },
+    };
+  };
+}

x-pack/test/lists_api_integration/common/ftr_provider_context.d.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
+
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

x-pack/test/lists_api_integration/common/services.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { services } from '../../api_integration/services';

x-pack/test/lists_api_integration/security_and_spaces/config.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { createTestConfig } from '../common/config';
+
+// eslint-disable-next-line import/no-default-export
+export default createTestConfig('security_and_spaces', {
+  disabledPlugins: [],
+  license: 'trial',
+  ssl: true,
+});