Add source map for every possible element in the Markdown preview

extensions/markdown-language-features/notebook/index.ts

@@ -3,9 +3,9 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-const MarkdownIt = require('markdown-it');
+const MarkdownIt: typeof import('markdown-it') = require('markdown-it');
 import * as DOMPurify from 'dompurify';
-import type * as markdownIt from 'markdown-it';
+import type * as MarkdownItToken from 'markdown-it/lib/token';
 import type { ActivationFunction } from 'vscode-notebook-renderer';
 
 const sanitizerOptions: DOMPurify.Config = {
@@ -196,12 +196,12 @@ export const activate: ActivationFunction<void> = (ctx) => {
 };
 
 
-function addNamedHeaderRendering(md: markdownIt.MarkdownIt): void {
+function addNamedHeaderRendering(md: InstanceType<typeof MarkdownIt>): void {
 	const slugCounter = new Map<string, number>();
 
 	const originalHeaderOpen = md.renderer.rules.heading_open;
-	md.renderer.rules.heading_open = (tokens: markdownIt.Token[], idx: number, options: any, env: any, self: any) => {
-		const title = tokens[idx + 1].children.reduce((acc: string, t: any) => acc + t.content, '');
+	md.renderer.rules.heading_open = (tokens: MarkdownItToken[], idx: number, options, env, self) => {
+		const title = tokens[idx + 1].children!.reduce<string>((acc, t) => acc + t.content, '');
 		let slug = slugFromHeading(title);
 
 		if (slugCounter.has(slug)) {
@@ -212,13 +212,12 @@ function addNamedHeaderRendering(md: markdownIt.MarkdownIt): void {
 			slugCounter.set(slug, 0);
 		}
 
-		tokens[idx].attrs = tokens[idx].attrs || [];
-		tokens[idx].attrs.push(['id', slug]);
+		tokens[idx].attrSet('id', slug);
 
 		if (originalHeaderOpen) {
 			return originalHeaderOpen(tokens, idx, options, env, self);
 		} else {
-			return self.renderToken(tokens, idx, options, env, self);
+			return self.renderToken(tokens, idx, options);
 		}
 	};
 

extensions/markdown-language-features/notebook/tsconfig.json

@@ -3,6 +3,7 @@
 	"compilerOptions": {
 		"outDir": "./dist/",
 		"jsx": "react",
+		"moduleResolution": "Node",
 		"module": "es2020",
 		"lib": [
 			"es2018",

extensions/markdown-language-features/package.json

@@ -352,18 +352,17 @@
     "watch-web": "npx webpack-cli --config extension-browser.webpack.config --mode none --watch --info-verbosity verbose"
   },
   "dependencies": {
-    "dompurify": "^2.3.1",
+    "dompurify": "^2.3.3",
     "highlight.js": "^10.4.1",
-    "markdown-it": "^12.0.3",
+    "markdown-it": "^12.2.0",
     "markdown-it-front-matter": "^0.2.1",
     "vscode-extension-telemetry": "0.4.2",
     "vscode-nls": "^5.0.0"
   },
   "devDependencies": {
-    "@types/dompurify": "^2.2.3",
-    "@types/highlight.js": "10.1.0",
+    "@types/dompurify": "^2.3.1",
     "@types/lodash.throttle": "^4.1.3",
-    "@types/markdown-it": "0.0.2",
+    "@types/markdown-it": "12.2.3",
     "@types/vscode-notebook-renderer": "^1.60.0",
     "@types/vscode-webview": "^1.57.0",
     "lodash.throttle": "^4.1.1"

extensions/markdown-language-features/src/features/foldingProvider.ts

@@ -3,13 +3,17 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Token } from 'markdown-it';
+import Token = require('markdown-it/lib/token');
 import * as vscode from 'vscode';
 import { MarkdownEngine } from '../markdownEngine';
 import { TableOfContentsProvider } from '../tableOfContentsProvider';
 
 const rangeLimit = 5000;
 
+interface MarkdownItTokenWithMap extends Token {
+	map: [number, number];
+}
+
 export default class MarkdownFoldingProvider implements vscode.FoldingRangeProvider {
 
 	constructor(
@@ -84,10 +88,14 @@ export default class MarkdownFoldingProvider implements vscode.FoldingRangeProvi
 const isStartRegion = (t: string) => /^\s*<!--\s*#?region\b.*-->/.test(t);
 const isEndRegion = (t: string) => /^\s*<!--\s*#?endregion\b.*-->/.test(t);
 
-const isRegionMarker = (token: Token) =>
-	token.type === 'html_block' && (isStartRegion(token.content) || isEndRegion(token.content));
+const isRegionMarker = (token: Token): token is MarkdownItTokenWithMap =>
+	!!token.map && token.type === 'html_block' && (isStartRegion(token.content) || isEndRegion(token.content));
+
+const isFoldableToken = (token: Token): token is MarkdownItTokenWithMap => {
+	if (!token.map) {
+		return false;
+	}
 
-const isFoldableToken = (token: Token): boolean => {
 	switch (token.type) {
 		case 'fence':
 		case 'list_item_open':

extensions/markdown-language-features/src/features/smartSelect.ts

@@ -2,11 +2,15 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-import { Token } from 'markdown-it';
+import Token = require('markdown-it/lib/token');
 import * as vscode from 'vscode';
 import { MarkdownEngine } from '../markdownEngine';
 import { TableOfContentsProvider, TocEntry } from '../tableOfContentsProvider';
 
+interface MarkdownItTokenWithMap extends Token {
+	map: [number, number];
+}
+
 export default class MarkdownSmartSelect implements vscode.SelectionRangeProvider {
 
 	constructor(
@@ -96,16 +100,16 @@ function createHeaderRange(header: TocEntry, isClosestHeaderToPosition: boolean,
 	}
 }
 
-function getBlockTokensForPosition(tokens: Token[], position: vscode.Position, parent?: vscode.SelectionRange): Token[] {
-	const enclosingTokens = tokens.filter(token => token.map && (token.map[0] <= position.line && token.map[1] > position.line) && (!parent || (token.map[0] >= parent.range.start.line && token.map[1] <= parent.range.end.line + 1)) && isBlockElement(token));
+function getBlockTokensForPosition(tokens: Token[], position: vscode.Position, parent?: vscode.SelectionRange): MarkdownItTokenWithMap[] {
+	const enclosingTokens = tokens.filter((token): token is MarkdownItTokenWithMap => !!token.map && (token.map[0] <= position.line && token.map[1] > position.line) && (!parent || (token.map[0] >= parent.range.start.line && token.map[1] <= parent.range.end.line + 1)) && isBlockElement(token));
 	if (enclosingTokens.length === 0) {
 		return [];
 	}
 	const sortedTokens = enclosingTokens.sort((token1, token2) => (token2.map[1] - token2.map[0]) - (token1.map[1] - token1.map[0]));
 	return sortedTokens;
 }
 
-function createBlockRange(block: Token, document: vscode.TextDocument, cursorLine: number, parent?: vscode.SelectionRange): vscode.SelectionRange | undefined {
+function createBlockRange(block: MarkdownItTokenWithMap, document: vscode.TextDocument, cursorLine: number, parent?: vscode.SelectionRange): vscode.SelectionRange | undefined {
 	if (block.type === 'fence') {
 		return createFencedRange(block, cursorLine, document, parent);
 	} else {
@@ -144,7 +148,7 @@ function createInlineRange(document: vscode.TextDocument, cursorPosition: vscode
 	return inlineCodeBlockSelection || linkSelection || comboSelection || boldSelection || italicSelection;
 }
 
-function createFencedRange(token: Token, cursorLine: number, document: vscode.TextDocument, parent?: vscode.SelectionRange): vscode.SelectionRange {
+function createFencedRange(token: MarkdownItTokenWithMap, cursorLine: number, document: vscode.TextDocument, parent?: vscode.SelectionRange): vscode.SelectionRange {
 	const startLine = token.map[0];
 	const endLine = token.map[1] - 1;
 	const onFenceLine = cursorLine === startLine || cursorLine === endLine;

extensions/markdown-language-features/src/markdownEngine.ts

@@ -3,7 +3,8 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { MarkdownIt, Token } from 'markdown-it';
+import MarkdownIt = require('markdown-it');
+import Token = require('markdown-it/lib/token');
 import * as vscode from 'vscode';
 import { MarkdownContributionProvider as MarkdownContributionProvider } from './markdownExtensions';
 import { Slugifier } from './slugify';
@@ -14,11 +15,34 @@ import { WebviewResourceProvider } from './util/resources';
 
 const UNICODE_NEWLINE_REGEX = /\u2028|\u2029/g;
 
-interface MarkdownItConfig {
-	readonly breaks: boolean;
-	readonly linkify: boolean;
-	readonly typographer: boolean;
-}
+/**
+ * Adds begin line index to the output via the 'data-line' data attribute.
+ */
+const pluginSourceMap: MarkdownIt.PluginSimple = (md): void => {
+	// Set the attribute on every possible token.
+	md.core.ruler.push('source_map_data_attribute', (state): void => {
+		for (const token of state.tokens) {
+			if (token.map && token.type !== 'inline') {
+				token.attrSet('data-line', String(token.map[0]));
+				token.attrJoin('class', 'code-line');
+			}
+		}
+	});
+
+	// The 'html_block' renderer doesn't respect `attrs`. We need to insert a marker.
+	const originalHtmlBlockRenderer = md.renderer.rules['html_block'];
+	if (originalHtmlBlockRenderer) {
+		md.renderer.rules['html_block'] = (tokens, idx, options, env, self) => (
+			`<div ${self.renderAttrs(tokens[idx])} ></div>\n` +
+			originalHtmlBlockRenderer(tokens, idx, options, env, self)
+		);
+	}
+};
+
+/**
+ * The markdown-it options that we expose in the settings.
+ */
+type MarkdownItConfig = Readonly<Required<Pick<MarkdownIt.Options, 'breaks' | 'linkify' | 'typographer'>>>;
 
 class TokenCache {
 	private cachedDocument?: {
@@ -85,7 +109,8 @@ export class MarkdownEngine {
 
 	private async getEngine(config: MarkdownItConfig): Promise<MarkdownIt> {
 		if (!this.md) {
-			this.md = import('markdown-it').then(async markdownIt => {
+			this.md = (async () => {
+				const markdownIt = await import('markdown-it');
 				let md: MarkdownIt = markdownIt(await getMarkdownOptions(() => md));
 
 				for (const plugin of this.contributionProvider.contributions.markdownItPlugins.values()) {
@@ -111,18 +136,15 @@ export class MarkdownEngine {
 					alt: ['paragraph', 'reference', 'blockquote', 'list']
 				});
 
-				for (const renderName of ['paragraph_open', 'heading_open', 'image', 'code_block', 'fence', 'blockquote_open', 'list_item_open']) {
-					this.addLineNumberRenderer(md, renderName);
-				}
-
 				this.addImageRenderer(md);
 				this.addFencedRenderer(md);
 				this.addLinkNormalizer(md);
 				this.addLinkValidator(md);
 				this.addNamedHeaders(md);
 				this.addLinkRenderer(md);
+				md.use(pluginSourceMap);
 				return md;
-			});
+			})();
 		}
 
 		const md = await this.md!;
@@ -170,7 +192,7 @@ export class MarkdownEngine {
 		};
 
 		const html = engine.renderer.render(tokens, {
-			...(engine as any).options,
+			...engine.options,
 			...config
 		}, env);
 
@@ -199,26 +221,9 @@ export class MarkdownEngine {
 		};
 	}
 
-	private addLineNumberRenderer(md: MarkdownIt, ruleName: string): void {
-		const original = md.renderer.rules[ruleName];
-		md.renderer.rules[ruleName] = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
-			const token = tokens[idx];
-			if (token.map && token.map.length) {
-				token.attrSet('data-line', token.map[0] + '');
-				token.attrJoin('class', 'code-line');
-			}
-
-			if (original) {
-				return original(tokens, idx, options, env, self);
-			} else {
-				return self.renderToken(tokens, idx, options, env, self);
-			}
-		};
-	}
-
 	private addImageRenderer(md: MarkdownIt): void {
 		const original = md.renderer.rules.image;
-		md.renderer.rules.image = (tokens: Token[], idx: number, options: any, env: RenderEnv, self: any) => {
+		md.renderer.rules.image = (tokens: Token[], idx: number, options, env: RenderEnv, self) => {
 			const token = tokens[idx];
 			token.attrJoin('class', 'loading');
 
@@ -237,20 +242,24 @@ export class MarkdownEngine {
 			if (original) {
 				return original(tokens, idx, options, env, self);
 			} else {
-				return self.renderToken(tokens, idx, options, env, self);
+				return self.renderToken(tokens, idx, options);
 			}
 		};
 	}
 
 	private addFencedRenderer(md: MarkdownIt): void {
 		const original = md.renderer.rules['fenced'];
-		md.renderer.rules['fenced'] = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
+		md.renderer.rules['fenced'] = (tokens: Token[], idx: number, options, env, self) => {
 			const token = tokens[idx];
 			if (token.map && token.map.length) {
 				token.attrJoin('class', 'hljs');
 			}
 
-			return original(tokens, idx, options, env, self);
+			if (original) {
+				return original(tokens, idx, options, env, self);
+			} else {
+				return self.renderToken(tokens, idx, options);
+			}
 		};
 	}
 
@@ -282,8 +291,8 @@ export class MarkdownEngine {
 
 	private addNamedHeaders(md: MarkdownIt): void {
 		const original = md.renderer.rules.heading_open;
-		md.renderer.rules.heading_open = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
-			const title = tokens[idx + 1].children.reduce((acc: string, t: any) => acc + t.content, '');
+		md.renderer.rules.heading_open = (tokens: Token[], idx: number, options, env, self) => {
+			const title = tokens[idx + 1].children!.reduce<string>((acc, t) => acc + t.content, '');
 			let slug = this.slugifier.fromHeading(title);
 
 			if (this._slugCount.has(slug.value)) {
@@ -294,30 +303,31 @@ export class MarkdownEngine {
 				this._slugCount.set(slug.value, 0);
 			}
 
-			tokens[idx].attrs = tokens[idx].attrs || [];
-			tokens[idx].attrs.push(['id', slug.value]);
+			tokens[idx].attrSet('id', slug.value);
 
 			if (original) {
 				return original(tokens, idx, options, env, self);
 			} else {
-				return self.renderToken(tokens, idx, options, env, self);
+				return self.renderToken(tokens, idx, options);
 			}
 		};
 	}
 
 	private addLinkRenderer(md: MarkdownIt): void {
-		const old_render = md.renderer.rules.link_open || ((tokens: Token[], idx: number, options: any, _env: any, self: any) => {
-			return self.renderToken(tokens, idx, options);
-		});
+		const original = md.renderer.rules.link_open;
 
-		md.renderer.rules.link_open = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
+		md.renderer.rules.link_open = (tokens: Token[], idx: number, options, env, self) => {
 			const token = tokens[idx];
-			const hrefIndex = token.attrIndex('href');
-			if (hrefIndex >= 0) {
-				const href = token.attrs[hrefIndex][1];
-				token.attrPush(['data-href', href]);
+			const href = token.attrGet('href');
+			// A string, including empty string, may be `href`.
+			if (typeof href === 'string') {
+				token.attrSet('data-href', href);
+			}
+			if (original) {
+				return original(tokens, idx, options, env, self);
+			} else {
+				return self.renderToken(tokens, idx, options);
 			}
-			return old_render(tokens, idx, options, env, self);
 		};
 	}
 
@@ -366,7 +376,7 @@ export class MarkdownEngine {
 	}
 }
 
-async function getMarkdownOptions(md: () => MarkdownIt) {
+async function getMarkdownOptions(md: () => MarkdownIt): Promise<MarkdownIt.Options> {
 	const hljs = await import('highlight.js');
 	return {
 		html: true,