Skip to content

Commit

Permalink
Add source map for every possible element in the Markdown preview (#1…
Browse files Browse the repository at this point in the history 
…34799)

* Update markdown-it and type definitions

* Refresh the source map mechanism in `markdownEngine.ts`
  • Loading branch information
@Lemmingh
Lemmingh committed Oct 12, 2021
1 parent d8f6711 commit d3ceb35
Show file tree
Hide file tree
Showing 7 changed files with 128 additions and 106 deletions.
15 changes: 7 additions & 8 deletions extensions/markdown-language-features/notebook/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

const MarkdownIt = require('markdown-it');
const MarkdownIt: typeof import('markdown-it') = require('markdown-it');
import * as DOMPurify from 'dompurify';
import type * as markdownIt from 'markdown-it';
import type * as MarkdownItToken from 'markdown-it/lib/token';
import type { ActivationFunction } from 'vscode-notebook-renderer';

const sanitizerOptions: DOMPurify.Config = {
Expand Down Expand Up @@ -196,12 +196,12 @@ export const activate: ActivationFunction<void> = (ctx) => {
};


function addNamedHeaderRendering(md: markdownIt.MarkdownIt): void {
function addNamedHeaderRendering(md: InstanceType<typeof MarkdownIt>): void {
const slugCounter = new Map<string, number>();

const originalHeaderOpen = md.renderer.rules.heading_open;
md.renderer.rules.heading_open = (tokens: markdownIt.Token[], idx: number, options: any, env: any, self: any) => {
const title = tokens[idx + 1].children.reduce((acc: string, t: any) => acc + t.content, '');
md.renderer.rules.heading_open = (tokens: MarkdownItToken[], idx: number, options, env, self) => {
const title = tokens[idx + 1].children!.reduce<string>((acc, t) => acc + t.content, '');
let slug = slugFromHeading(title);

if (slugCounter.has(slug)) {
Expand All @@ -212,13 +212,12 @@ function addNamedHeaderRendering(md: markdownIt.MarkdownIt): void {
slugCounter.set(slug, 0);
}

tokens[idx].attrs = tokens[idx].attrs || [];
tokens[idx].attrs.push(['id', slug]);
tokens[idx].attrSet('id', slug);

if (originalHeaderOpen) {
return originalHeaderOpen(tokens, idx, options, env, self);
} else {
return self.renderToken(tokens, idx, options, env, self);
return self.renderToken(tokens, idx, options);
}
};

Expand Down
1 change: 1 addition & 0 deletions extensions/markdown-language-features/notebook/tsconfig.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"compilerOptions": {
"outDir": "./dist/",
"jsx": "react",
"moduleResolution": "Node",
"module": "es2020",
"lib": [
"es2018",
Expand Down
9 changes: 4 additions & 5 deletions extensions/markdown-language-features/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,18 +352,17 @@
"watch-web": "npx webpack-cli --config extension-browser.webpack.config --mode none --watch --info-verbosity verbose"
},
"dependencies": {
"dompurify": "^2.3.1",
"dompurify": "^2.3.3",
"highlight.js": "^10.4.1",
"markdown-it": "^12.0.3",
"markdown-it": "^12.2.0",
"markdown-it-front-matter": "^0.2.1",
"vscode-extension-telemetry": "0.4.2",
"vscode-nls": "^5.0.0"
},
"devDependencies": {
"@types/dompurify": "^2.2.3",
"@types/highlight.js": "10.1.0",
"@types/dompurify": "^2.3.1",
"@types/lodash.throttle": "^4.1.3",
"@types/markdown-it": "0.0.2",
"@types/markdown-it": "12.2.3",
"@types/vscode-notebook-renderer": "^1.60.0",
"@types/vscode-webview": "^1.57.0",
"lodash.throttle": "^4.1.1"
Expand Down
16 changes: 12 additions & 4 deletions extensions/markdown-language-features/src/features/foldingProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { Token } from 'markdown-it';
import Token = require('markdown-it/lib/token');
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { TableOfContentsProvider } from '../tableOfContentsProvider';

const rangeLimit = 5000;

interface MarkdownItTokenWithMap extends Token {
map: [number, number];
}

export default class MarkdownFoldingProvider implements vscode.FoldingRangeProvider {

constructor(
Expand Down Expand Up @@ -84,10 +88,14 @@ export default class MarkdownFoldingProvider implements vscode.FoldingRangeProvi
const isStartRegion = (t: string) => /^\s*<!--\s*#?region\b.*-->/.test(t);
const isEndRegion = (t: string) => /^\s*<!--\s*#?endregion\b.*-->/.test(t);

const isRegionMarker = (token: Token) =>
token.type === 'html_block' && (isStartRegion(token.content) || isEndRegion(token.content));
const isRegionMarker = (token: Token): token is MarkdownItTokenWithMap =>
!!token.map && token.type === 'html_block' && (isStartRegion(token.content) || isEndRegion(token.content));

const isFoldableToken = (token: Token): token is MarkdownItTokenWithMap => {
if (!token.map) {
return false;
}

const isFoldableToken = (token: Token): boolean => {
switch (token.type) {
case 'fence':
case 'list_item_open':
Expand Down
14 changes: 9 additions & 5 deletions extensions/markdown-language-features/src/features/smartSelect.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { Token } from 'markdown-it';
import Token = require('markdown-it/lib/token');
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { TableOfContentsProvider, TocEntry } from '../tableOfContentsProvider';

interface MarkdownItTokenWithMap extends Token {
map: [number, number];
}

export default class MarkdownSmartSelect implements vscode.SelectionRangeProvider {

constructor(
Expand Down Expand Up @@ -96,16 +100,16 @@ function createHeaderRange(header: TocEntry, isClosestHeaderToPosition: boolean,
}
}

function getBlockTokensForPosition(tokens: Token[], position: vscode.Position, parent?: vscode.SelectionRange): Token[] {
const enclosingTokens = tokens.filter(token => token.map && (token.map[0] <= position.line && token.map[1] > position.line) && (!parent || (token.map[0] >= parent.range.start.line && token.map[1] <= parent.range.end.line + 1)) && isBlockElement(token));
function getBlockTokensForPosition(tokens: Token[], position: vscode.Position, parent?: vscode.SelectionRange): MarkdownItTokenWithMap[] {
const enclosingTokens = tokens.filter((token): token is MarkdownItTokenWithMap => !!token.map && (token.map[0] <= position.line && token.map[1] > position.line) && (!parent || (token.map[0] >= parent.range.start.line && token.map[1] <= parent.range.end.line + 1)) && isBlockElement(token));
if (enclosingTokens.length === 0) {
return [];
}
const sortedTokens = enclosingTokens.sort((token1, token2) => (token2.map[1] - token2.map[0]) - (token1.map[1] - token1.map[0]));
return sortedTokens;
}

function createBlockRange(block: Token, document: vscode.TextDocument, cursorLine: number, parent?: vscode.SelectionRange): vscode.SelectionRange | undefined {
function createBlockRange(block: MarkdownItTokenWithMap, document: vscode.TextDocument, cursorLine: number, parent?: vscode.SelectionRange): vscode.SelectionRange | undefined {
if (block.type === 'fence') {
return createFencedRange(block, cursorLine, document, parent);
} else {
Expand Down Expand Up @@ -144,7 +148,7 @@ function createInlineRange(document: vscode.TextDocument, cursorPosition: vscode
return inlineCodeBlockSelection || linkSelection || comboSelection || boldSelection || italicSelection;
}

function createFencedRange(token: Token, cursorLine: number, document: vscode.TextDocument, parent?: vscode.SelectionRange): vscode.SelectionRange {
function createFencedRange(token: MarkdownItTokenWithMap, cursorLine: number, document: vscode.TextDocument, parent?: vscode.SelectionRange): vscode.SelectionRange {
const startLine = token.map[0];
const endLine = token.map[1] - 1;
const onFenceLine = cursorLine === startLine || cursorLine === endLine;
Expand Down
108 changes: 59 additions & 49 deletions extensions/markdown-language-features/src/markdownEngine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { MarkdownIt, Token } from 'markdown-it';
import MarkdownIt = require('markdown-it');
import Token = require('markdown-it/lib/token');
import * as vscode from 'vscode';
import { MarkdownContributionProvider as MarkdownContributionProvider } from './markdownExtensions';
import { Slugifier } from './slugify';
Expand All @@ -14,11 +15,34 @@ import { WebviewResourceProvider } from './util/resources';

const UNICODE_NEWLINE_REGEX = /\u2028|\u2029/g;

interface MarkdownItConfig {
readonly breaks: boolean;
readonly linkify: boolean;
readonly typographer: boolean;
}
/**
* Adds begin line index to the output via the 'data-line' data attribute.
*/
const pluginSourceMap: MarkdownIt.PluginSimple = (md): void => {
// Set the attribute on every possible token.
md.core.ruler.push('source_map_data_attribute', (state): void => {
for (const token of state.tokens) {
if (token.map && token.type !== 'inline') {
token.attrSet('data-line', String(token.map[0]));
token.attrJoin('class', 'code-line');
}
}
});

// The 'html_block' renderer doesn't respect `attrs`. We need to insert a marker.
const originalHtmlBlockRenderer = md.renderer.rules['html_block'];
if (originalHtmlBlockRenderer) {
md.renderer.rules['html_block'] = (tokens, idx, options, env, self) => (
`<div ${self.renderAttrs(tokens[idx])} ></div>\n` +
originalHtmlBlockRenderer(tokens, idx, options, env, self)
);
}
};

/**
* The markdown-it options that we expose in the settings.
*/
type MarkdownItConfig = Readonly<Required<Pick<MarkdownIt.Options, 'breaks' | 'linkify' | 'typographer'>>>;

class TokenCache {
private cachedDocument?: {
Expand Down Expand Up @@ -85,7 +109,8 @@ export class MarkdownEngine {

private async getEngine(config: MarkdownItConfig): Promise<MarkdownIt> {
if (!this.md) {
this.md = import('markdown-it').then(async markdownIt => {
this.md = (async () => {
const markdownIt = await import('markdown-it');
let md: MarkdownIt = markdownIt(await getMarkdownOptions(() => md));

for (const plugin of this.contributionProvider.contributions.markdownItPlugins.values()) {
Expand All @@ -111,18 +136,15 @@ export class MarkdownEngine {
alt: ['paragraph', 'reference', 'blockquote', 'list']
});

for (const renderName of ['paragraph_open', 'heading_open', 'image', 'code_block', 'fence', 'blockquote_open', 'list_item_open']) {
this.addLineNumberRenderer(md, renderName);
}

this.addImageRenderer(md);
this.addFencedRenderer(md);
this.addLinkNormalizer(md);
this.addLinkValidator(md);
this.addNamedHeaders(md);
this.addLinkRenderer(md);
md.use(pluginSourceMap);
return md;
});
})();
}

const md = await this.md!;
Expand Down Expand Up @@ -170,7 +192,7 @@ export class MarkdownEngine {
};

const html = engine.renderer.render(tokens, {
...(engine as any).options,
...engine.options,
...config
}, env);

Expand Down Expand Up @@ -199,26 +221,9 @@ export class MarkdownEngine {
};
}

private addLineNumberRenderer(md: MarkdownIt, ruleName: string): void {
const original = md.renderer.rules[ruleName];
md.renderer.rules[ruleName] = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
const token = tokens[idx];
if (token.map && token.map.length) {
token.attrSet('data-line', token.map[0] + '');
token.attrJoin('class', 'code-line');
}

if (original) {
return original(tokens, idx, options, env, self);
} else {
return self.renderToken(tokens, idx, options, env, self);
}
};
}

private addImageRenderer(md: MarkdownIt): void {
const original = md.renderer.rules.image;
md.renderer.rules.image = (tokens: Token[], idx: number, options: any, env: RenderEnv, self: any) => {
md.renderer.rules.image = (tokens: Token[], idx: number, options, env: RenderEnv, self) => {
const token = tokens[idx];
token.attrJoin('class', 'loading');

Expand All @@ -237,20 +242,24 @@ export class MarkdownEngine {
if (original) {
return original(tokens, idx, options, env, self);
} else {
return self.renderToken(tokens, idx, options, env, self);
return self.renderToken(tokens, idx, options);
}
};
}

private addFencedRenderer(md: MarkdownIt): void {
const original = md.renderer.rules['fenced'];
md.renderer.rules['fenced'] = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
md.renderer.rules['fenced'] = (tokens: Token[], idx: number, options, env, self) => {
const token = tokens[idx];
if (token.map && token.map.length) {
token.attrJoin('class', 'hljs');
}

return original(tokens, idx, options, env, self);
if (original) {
return original(tokens, idx, options, env, self);
} else {
return self.renderToken(tokens, idx, options);
}
};
}

Expand Down Expand Up @@ -282,8 +291,8 @@ export class MarkdownEngine {

private addNamedHeaders(md: MarkdownIt): void {
const original = md.renderer.rules.heading_open;
md.renderer.rules.heading_open = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
const title = tokens[idx + 1].children.reduce((acc: string, t: any) => acc + t.content, '');
md.renderer.rules.heading_open = (tokens: Token[], idx: number, options, env, self) => {
const title = tokens[idx + 1].children!.reduce<string>((acc, t) => acc + t.content, '');
let slug = this.slugifier.fromHeading(title);

if (this._slugCount.has(slug.value)) {
Expand All @@ -294,30 +303,31 @@ export class MarkdownEngine {
this._slugCount.set(slug.value, 0);
}

tokens[idx].attrs = tokens[idx].attrs || [];
tokens[idx].attrs.push(['id', slug.value]);
tokens[idx].attrSet('id', slug.value);

if (original) {
return original(tokens, idx, options, env, self);
} else {
return self.renderToken(tokens, idx, options, env, self);
return self.renderToken(tokens, idx, options);
}
};
}

private addLinkRenderer(md: MarkdownIt): void {
const old_render = md.renderer.rules.link_open || ((tokens: Token[], idx: number, options: any, _env: any, self: any) => {
return self.renderToken(tokens, idx, options);
});
const original = md.renderer.rules.link_open;

md.renderer.rules.link_open = (tokens: Token[], idx: number, options: any, env: any, self: any) => {
md.renderer.rules.link_open = (tokens: Token[], idx: number, options, env, self) => {
const token = tokens[idx];
const hrefIndex = token.attrIndex('href');
if (hrefIndex >= 0) {
const href = token.attrs[hrefIndex][1];
token.attrPush(['data-href', href]);
const href = token.attrGet('href');
// A string, including empty string, may be `href`.
if (typeof href === 'string') {
token.attrSet('data-href', href);
}
if (original) {
return original(tokens, idx, options, env, self);
} else {
return self.renderToken(tokens, idx, options);
}
return old_render(tokens, idx, options, env, self);
};
}

Expand Down Expand Up @@ -366,7 +376,7 @@ export class MarkdownEngine {
}
}

async function getMarkdownOptions(md: () => MarkdownIt) {
async function getMarkdownOptions(md: () => MarkdownIt): Promise<MarkdownIt.Options> {
const hljs = await import('highlight.js');
return {
html: true,
Expand Down
Loading

0 comments on commit d3ceb35

Please sign in to comment.