Automatically generated Api documentation (#86232)

* auto generated mdx api doc system

* Fix README.md

* update core api docs after master merge

* clean up signature

* Update packages/kbn-dev-utils/src/plugins/parse_kibana_platform_plugin.ts

Co-authored-by: Spencer <email@spalger.com>

* migrate to docs-util package

* Remove bad links

* fix ref to release-notes and add extra dats service folder

* update name change

* Update packages/kbn-docs-utils/src/api_docs/build_api_declarations/get_type_kind.ts

Co-authored-by: Brandon Kobel <brandon.kobel@gmail.com>

* Update packages/kbn-docs-utils/src/api_docs/utils.ts

Co-authored-by: Brandon Kobel <brandon.kobel@gmail.com>

* review updates 1

* review feedback updates round 1

* Small refactor of extractImportReferences

* Review feedback updates 2

* Review update 3 plus support for links in class interface heritage clause

* debug failing test on ci only

* Escape regex directory path

* Update packages/kbn-docs-utils/src/api_docs/build_api_declarations/utils.ts

Co-authored-by: Spencer <email@spalger.com>

* fix for commit suggestion

Co-authored-by: Spencer <email@spalger.com>
Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Brandon Kobel <brandon.kobel@gmail.com>
Co-authored-by: kobelb <brandon.kobel@elastic.co>

.gitignore

@@ -1,5 +1,6 @@
 .aws-config.json
 .signing-config.json
+/api_docs
 .ackrc
 /.es
 /.chromium

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md

@@ -28,6 +28,7 @@ Should never be used in code outside of Core but is exported for documentation p
 |  [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md) | <code>readonly string[]</code> | List of plugin ids that this plugin's UI code imports modules from that are not in <code>requiredPlugins</code>. |
 |  [requiredPlugins](./kibana-plugin-core-server.pluginmanifest.requiredplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. |
 |  [server](./kibana-plugin-core-server.pluginmanifest.server.md) | <code>boolean</code> | Specifies whether plugin includes some server-side specific functionality. |
+|  [serviceFolders](./kibana-plugin-core-server.pluginmanifest.servicefolders.md) | <code>readonly string[]</code> | Only used for the automatically generated API documentation. Specifying service folders will cause your plugin API reference to be broken up into sub sections. |
 |  [ui](./kibana-plugin-core-server.pluginmanifest.ui.md) | <code>boolean</code> | Specifies whether plugin includes some client/browser specific functionality that should be included into client bundle via <code>public/ui_plugin.js</code> file. |
 |  [version](./kibana-plugin-core-server.pluginmanifest.version.md) | <code>string</code> | Version of the plugin. |
 

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.servicefolders.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) &gt; [serviceFolders](./kibana-plugin-core-server.pluginmanifest.servicefolders.md)
+
+## PluginManifest.serviceFolders property
+
+Only used for the automatically generated API documentation. Specifying service folders will cause your plugin API reference to be broken up into sub sections.
+
+<b>Signature:</b>
+
+```typescript
+readonly serviceFolders?: readonly string[];
+```

package.json

@@ -47,6 +47,7 @@
     "test:ftr:runner": "node scripts/functional_test_runner",
     "checkLicenses": "node scripts/check_licenses --dev",
     "build": "node scripts/build --all-platforms",
+    "build:apidocs": "node scripts/build_api_docs",
     "start": "node scripts/kibana --dev",
     "debug": "node --nolazy --inspect scripts/kibana --dev",
     "debug-break": "node --nolazy --inspect-brk scripts/kibana --dev",
@@ -367,7 +368,7 @@
     "@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
     "@kbn/plugin-helpers": "link:packages/kbn-plugin-helpers",
     "@kbn/pm": "link:packages/kbn-pm",
-    "@kbn/release-notes": "link:packages/kbn-release-notes",
+    "@kbn/docs-utils": "link:packages/kbn-docs-utils",
     "@kbn/storybook": "link:packages/kbn-storybook",
     "@kbn/telemetry-tools": "link:packages/kbn-telemetry-tools",
     "@kbn/test": "link:packages/kbn-test",
@@ -821,6 +822,7 @@
     "tinycolor2": "1.4.1",
     "topojson-client": "3.0.0",
     "ts-loader": "^7.0.5",
+    "ts-morph": "^9.1.0",
     "tsd": "^0.13.1",
     "typescript": "4.1.3",
     "typescript-fsa": "^3.0.0",

packages/kbn-dev-utils/src/plugins/parse_kibana_platform_plugin.ts

@@ -29,6 +29,7 @@ interface Manifest {
   server: boolean;
   kibanaVersion: string;
   version: string;
+  serviceFolders: readonly string[];
   requiredPlugins: readonly string[];
   optionalPlugins: readonly string[];
   requiredBundles: readonly string[];
@@ -64,6 +65,7 @@ export function parseKibanaPlatformPlugin(manifestPath: string): KibanaPlatformP
       id: manifest.id,
       version: manifest.version,
       kibanaVersion: manifest.kibanaVersion || manifest.version,
+      serviceFolders: manifest.serviceFolders || [],
       requiredPlugins: isValidDepsDeclaration(manifest.requiredPlugins, 'requiredPlugins'),
       optionalPlugins: isValidDepsDeclaration(manifest.optionalPlugins, 'optionalPlugins'),
       requiredBundles: isValidDepsDeclaration(manifest.requiredBundles, 'requiredBundles'),

packages/kbn-release-notes/jest.config.js → packages/kbn-docs-utils/jest.config.js

@@ -9,5 +9,5 @@
 module.exports = {
   preset: '@kbn/test',
   rootDir: '../..',
-  roots: ['<rootDir>/packages/kbn-release-notes'],
+  roots: ['<rootDir>/packages/kbn-docs-utils'],
 };

packages/kbn-release-notes/package.json → packages/kbn-docs-utils/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@kbn/release-notes",
+  "name": "@kbn/docs-utils",
   "version": "1.0.0",
   "license": "SSPL-1.0 OR Elastic License 2.0",
   "private": "true",

packages/kbn-docs-utils/src/api_docs/README.md

@@ -0,0 +1,12 @@
+# Autogenerated API documentation
+
+[RFC](../../../rfcs/text/0014_api_documentation.md)
+
+This is an experimental api documentation system that is managed by the Kibana Tech Leads until
+we determine the value of such a system and what kind of maintenance burder it will incur.
+
+To generate the docs run 
+
+```
+node scripts/build_api_docs
+```

packages/kbn-docs-utils/src/api_docs/build_api_declarations/buid_api_declaration.test.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import { Project, Node } from 'ts-morph';
+import { ToolingLog, KibanaPlatformPlugin } from '@kbn/dev-utils';
+
+import { TypeKind, ApiScope } from '../types';
+import { getKibanaPlatformPlugin } from '../tests/kibana_platform_plugin_mock';
+import { getDeclarationNodesForPluginScope } from '../get_declaration_nodes_for_plugin';
+import { buildApiDeclaration } from './build_api_declaration';
+import { isNamedNode } from '../tsmorph_utils';
+
+const log = new ToolingLog({
+  level: 'debug',
+  writeTo: process.stdout,
+});
+
+let nodes: Node[];
+let plugins: KibanaPlatformPlugin[];
+
+function getNodeName(node: Node): string {
+  return isNamedNode(node) ? node.getName() : '';
+}
+
+beforeAll(() => {
+  const tsConfigFilePath = Path.resolve(__dirname, '../tests/__fixtures__/src/tsconfig.json');
+  const project = new Project({
+    tsConfigFilePath,
+  });
+
+  plugins = [getKibanaPlatformPlugin('pluginA')];
+
+  nodes = getDeclarationNodesForPluginScope(project, plugins[0], ApiScope.CLIENT, log);
+});
+
+it('Test number primitive doc def', () => {
+  const node = nodes.find((n) => getNodeName(n) === 'aNum');
+  expect(node).toBeDefined();
+  const def = buildApiDeclaration(node!, plugins, log, plugins[0].manifest.id, ApiScope.CLIENT);
+
+  expect(def.type).toBe(TypeKind.NumberKind);
+});
+
+it('Function type is exported as type with signature', () => {
+  const node = nodes.find((n) => getNodeName(n) === 'FnWithGeneric');
+  expect(node).toBeDefined();
+  const def = buildApiDeclaration(node!, plugins, log, plugins[0].manifest.id, ApiScope.CLIENT);
+  expect(def).toBeDefined();
+  expect(def?.type).toBe(TypeKind.TypeKind);
+  expect(def?.signature?.length).toBeGreaterThan(0);
+});
+
+it('Test Interface Kind doc def', () => {
+  const node = nodes.find((n) => getNodeName(n) === 'ExampleInterface');
+  expect(node).toBeDefined();
+  const def = buildApiDeclaration(node!, plugins, log, plugins[0].manifest.id, ApiScope.CLIENT);
+
+  expect(def.type).toBe(TypeKind.InterfaceKind);
+  expect(def.children).toBeDefined();
+  expect(def.children!.length).toBe(3);
+});
+
+it('Test union export', () => {
+  const node = nodes.find((n) => getNodeName(n) === 'aUnionProperty');
+  expect(node).toBeDefined();
+  const def = buildApiDeclaration(node!, plugins, log, plugins[0].manifest.id, ApiScope.CLIENT);
+  expect(def.type).toBe(TypeKind.CompoundTypeKind);
+});
+
+it('Function inside interface has a label', () => {
+  const node = nodes.find((n) => getNodeName(n) === 'ExampleInterface');
+  expect(node).toBeDefined();
+  const def = buildApiDeclaration(node!, plugins, log, plugins[0].manifest.id, ApiScope.CLIENT);
+
+  const fn = def!.children?.find((c) => c.label === 'aFn');
+  expect(fn).toBeDefined();
+  expect(fn?.label).toBe('aFn');
+  expect(fn?.type).toBe(TypeKind.FunctionKind);
+});

packages/kbn-docs-utils/src/api_docs/build_api_declarations/build_api_declaration.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Node } from 'ts-morph';
+import { ToolingLog, KibanaPlatformPlugin } from '@kbn/dev-utils';
+import { buildClassDec } from './build_class_dec';
+import { buildFunctionDec } from './build_function_dec';
+import { getCommentsFromNode } from './js_doc_utils';
+import { isNamedNode } from '../tsmorph_utils';
+import { AnchorLink, ApiDeclaration } from '../types';
+import { buildVariableDec } from './build_variable_dec';
+import { getApiSectionId } from '../utils';
+import { getSourceForNode } from './utils';
+import { buildTypeLiteralDec } from './build_type_literal_dec';
+import { ApiScope } from '../types';
+import { getSignature } from './get_signature';
+import { buildInterfaceDec } from './build_interface_dec';
+import { getTypeKind } from './get_type_kind';
+
+/**
+ * A potentially recursive function, depending on the node type, that builds a JSON like structure
+ * that can be passed to the elastic-docs component for rendering as an API. Nodes like classes,
+ * interfaces, objects and functions will have children for their properties, members and parameters.
+ *
+ * @param node The ts-morph node to build an ApiDeclaration for.
+ * @param plugins The list of plugins registered is used for building cross plugin links by looking up
+ * the plugin by import path. We could accomplish the same thing via a regex on the import path, but this lets us
+ * decouple plugin path from plugin id.
+ * @param log Logs messages to console.
+ * @param pluginName The name of the plugin this declaration belongs to.
+ * @param scope The scope this declaration belongs to (server, public, or common).
+ * @param parentApiId If this declaration is nested inside another declaration, it should have a parent id. This
+ * is used to create the anchor link to this API item.
+ * @param name An optional name to pass through which will be used instead of node.getName, if it
+ * exists. For some types, like Parameters, the name comes on the parent node, but we want the doc def
+ * to be built from the TypedNode
+ */
+export function buildApiDeclaration(
+  node: Node,
+  plugins: KibanaPlatformPlugin[],
+  log: ToolingLog,
+  pluginName: string,
+  scope: ApiScope,
+  parentApiId?: string,
+  name?: string
+): ApiDeclaration {
+  const apiName = name ? name : isNamedNode(node) ? node.getName() : 'Unnamed';
+  log.debug(`Building API Declaration for ${apiName} of kind ${node.getKindName()}`);
+  const apiId = parentApiId ? parentApiId + '.' + apiName : apiName;
+  const anchorLink: AnchorLink = { scope, pluginName, apiName: apiId };
+
+  if (Node.isClassDeclaration(node)) {
+    return buildClassDec(node, plugins, anchorLink, log);
+  } else if (Node.isInterfaceDeclaration(node)) {
+    return buildInterfaceDec(node, plugins, anchorLink, log);
+  } else if (
+    Node.isMethodSignature(node) ||
+    Node.isFunctionDeclaration(node) ||
+    Node.isMethodDeclaration(node) ||
+    Node.isConstructorDeclaration(node)
+  ) {
+    return buildFunctionDec(node, plugins, anchorLink, log);
+  } else if (
+    Node.isPropertySignature(node) ||
+    Node.isPropertyDeclaration(node) ||
+    Node.isShorthandPropertyAssignment(node) ||
+    Node.isPropertyAssignment(node) ||
+    Node.isVariableDeclaration(node)
+  ) {
+    return buildVariableDec(node, plugins, anchorLink, log);
+  } else if (Node.isTypeLiteralNode(node)) {
+    return buildTypeLiteralDec(node, plugins, anchorLink, log, apiName);
+  }
+
+  return {
+    id: getApiSectionId(anchorLink),
+    type: getTypeKind(node),
+    label: apiName,
+    description: getCommentsFromNode(node),
+    source: getSourceForNode(node),
+    signature: getSignature(node, plugins, log),
+  };
+}

packages/kbn-docs-utils/src/api_docs/build_api_declarations/build_arrow_fn_dec.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ToolingLog, KibanaPlatformPlugin } from '@kbn/dev-utils';
+
+import {
+  ArrowFunction,
+  VariableDeclaration,
+  PropertyDeclaration,
+  PropertySignature,
+  ShorthandPropertyAssignment,
+  PropertyAssignment,
+} from 'ts-morph';
+import { getApiSectionId } from '../utils';
+import { getCommentsFromNode } from './js_doc_utils';
+import { AnchorLink, TypeKind } from '../types';
+import { getSourceForNode } from './utils';
+import { buildApiDecsForParameters } from './build_parameter_decs';
+import { getSignature } from './get_signature';
+import { getJSDocReturnTagComment } from './js_doc_utils';
+
+/**
+ * Arrow functions are handled differently than regular functions because you need the arrow function
+ * initializer as well as the node. The initializer is where the parameters are grabbed from and the
+ * signature, while the node has the comments and name.
+ *
+ * @param node
+ * @param initializer
+ * @param plugins
+ * @param anchorLink
+ * @param log
+ */
+export function getArrowFunctionDec(
+  node:
+    | VariableDeclaration
+    | PropertyDeclaration
+    | PropertySignature
+    | ShorthandPropertyAssignment
+    | PropertyAssignment,
+  initializer: ArrowFunction,
+  plugins: KibanaPlatformPlugin[],
+  anchorLink: AnchorLink,
+  log: ToolingLog
+) {
+  log.debug(
+    `Getting Arrow Function doc def for node ${node.getName()} of kind ${node.getKindName()}`
+  );
+  return {
+    id: getApiSectionId(anchorLink),
+    type: TypeKind.FunctionKind,
+    children: buildApiDecsForParameters(initializer.getParameters(), plugins, anchorLink, log),
+    signature: getSignature(initializer, plugins, log),
+    description: getCommentsFromNode(node),
+    label: node.getName(),
+    source: getSourceForNode(node),
+    returnComment: getJSDocReturnTagComment(node),
+  };
+}

packages/kbn-docs-utils/src/api_docs/build_api_declarations/build_class_dec.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ToolingLog, KibanaPlatformPlugin } from '@kbn/dev-utils';
+import { ClassDeclaration } from 'ts-morph';
+import { AnchorLink, ApiDeclaration, TypeKind } from '../types';
+import { getCommentsFromNode } from './js_doc_utils';
+import { buildApiDeclaration } from './build_api_declaration';
+import { getSourceForNode, isPrivate } from './utils';
+import { getApiSectionId } from '../utils';
+import { getSignature } from './get_signature';
+
+export function buildClassDec(
+  node: ClassDeclaration,
+  plugins: KibanaPlatformPlugin[],
+  anchorLink: AnchorLink,
+  log: ToolingLog
+): ApiDeclaration {
+  return {
+    id: getApiSectionId(anchorLink),
+    type: TypeKind.ClassKind,
+    label: node.getName() || 'Missing label',
+    description: getCommentsFromNode(node),
+    signature: getSignature(node, plugins, log),
+    children: node.getMembers().reduce((acc, m) => {
+      if (!isPrivate(m)) {
+        acc.push(
+          buildApiDeclaration(
+            m,
+            plugins,
+            log,
+            anchorLink.pluginName,
+            anchorLink.scope,
+            anchorLink.apiName
+          )
+        );
+      }
+      return acc;
+    }, [] as ApiDeclaration[]),
+    source: getSourceForNode(node),
+  };
+}