From 27f00c678af6ad9a2af85da2645d484a576caa0c Mon Sep 17 00:00:00 2001 From: Puru Date: Sat, 11 May 2024 01:22:36 +0530 Subject: [PATCH 1/2] Push --- packages/site-kit/package.json | 2 + .../site-kit/src/lib/markdown/renderer.js | 65 ++++++------------- packages/site-kit/src/lib/markdown/shiki.js | 63 ++++++++++++++++++ packages/site-kit/src/lib/styles/code.css | 2 +- packages/site-kit/src/lib/styles/text.css | 10 +++ pnpm-lock.yaml | 50 ++++++++++++++ 6 files changed, 147 insertions(+), 45 deletions(-) create mode 100644 packages/site-kit/src/lib/markdown/shiki.js diff --git a/packages/site-kit/package.json b/packages/site-kit/package.json index 9c019914..f4739ca2 100644 --- a/packages/site-kit/package.json +++ b/packages/site-kit/package.json @@ -23,6 +23,7 @@ "svelte-persisted-store": "^0.9.2" }, "devDependencies": { + "@shikijs/twoslash": "^1.5.1", "@sveltejs/kit": "^2.5.7", "@sveltejs/package": "^2.3.1", "@types/node": "^20.12.11", @@ -30,6 +31,7 @@ "magic-string": "^0.30.10", "marked": "^12.0.2", "prettier": "^3.2.5", + "shiki": "^1.5.1", "shiki-twoslash": "^3.1.2", "svelte": "^4.2.16", "typescript": "^5.4.5", diff --git a/packages/site-kit/src/lib/markdown/renderer.js b/packages/site-kit/src/lib/markdown/renderer.js index ae5fd2d0..c4d24455 100644 --- a/packages/site-kit/src/lib/markdown/renderer.js +++ b/packages/site-kit/src/lib/markdown/renderer.js @@ -3,6 +3,7 @@ import { createHash } from 'node:crypto'; import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import path from 'node:path'; import ts from 'typescript'; +import { highlight } from './shiki.js'; import { SHIKI_LANGUAGE_MAP, escape, normalizeSlugify, transform } from './utils.js'; /** @@ -18,9 +19,6 @@ const METADATA_REGEX = /** @type {Map} */ const CACHE_MAP = new Map(); -/** @type {import('shiki-twoslash')} */ -let twoslash_module; - /** @type {import('prettier')} */ let prettier_module; @@ -112,11 +110,8 @@ export async function render_content_markdown( body, { twoslashBanner, modules = [], cacheCodeSnippets = true, resolveTypeLinks } = {} ) { - twoslash_module ??= await import('shiki-twoslash'); prettier_module ??= await import('prettier'); - const highlighter = await twoslash_module.createShikiHighlighter({ theme: 'css-variables' }); - const { type_links, type_regex } = create_type_links(modules, resolveTypeLinks); const SNIPPET_CACHE = await create_snippet_cache(cacheCodeSnippets); @@ -144,7 +139,6 @@ export async function render_content_markdown( let html = syntax_highlight({ filename, - highlighter, language, source, twoslashBanner, @@ -199,7 +193,7 @@ export async function render_content_markdown( ? `${name}` : ''; return `${prefix || ''}${link}`; - }) + }) : text) + '' ); @@ -454,8 +448,8 @@ export async function convert_to_ts(js_code, indent = '', offset = '') { js_code.includes('---cut---') ? js_code.indexOf('\n', js_code.indexOf('---cut---')) + 1 : js_code.includes('/// file:') - ? js_code.indexOf('\n', js_code.indexOf('/// file:')) + 1 - : 0 + ? js_code.indexOf('\n', js_code.indexOf('/// file:')) + 1 + : 0 ); code.appendLeft(insertion_point, offset + import_statements + '\n'); } @@ -930,25 +924,18 @@ function replace_blank_lines(html) { * source: string, * filename: string, * language: string, - * highlighter: ReturnType * twoslashBanner?: TwoslashBanner * options: SnippetOptions * }} param0 */ -function syntax_highlight({ source, filename, language, highlighter, twoslashBanner, options }) { +function syntax_highlight({ source, filename, language, twoslashBanner, options }) { let html = ''; - if (/^(dts|yaml|yml)/.test(language)) { - html = replace_blank_lines( - twoslash_module.renderCodeToHTML( - source, - language === 'dts' ? 'ts' : language, - { twoslash: false }, - { themeName: 'css-variables' }, - highlighter - ) - ); - } else if (/^(js|ts)/.test(language)) { + // console.log({ source, language }); + + if (/^(dts|yaml|yml)$/.test(language)) { + html = replace_blank_lines(highlight(source, language === 'dts' ? 'ts' : language)); + } else if (/^(js|ts)$/.test(language)) { try { const banner = twoslashBanner?.(filename, source, language, options); @@ -963,24 +950,7 @@ function syntax_highlight({ source, filename, language, highlighter, twoslashBan } } - const twoslash = twoslash_module.runTwoSlash(source, language, { - defaultCompilerOptions: { - allowJs: true, - checkJs: true, - target: ts.ScriptTarget.ES2022, - types: ['svelte', '@sveltejs/kit'] - } - }); - - html = twoslash_module.renderCodeToHTML( - twoslash.code, - 'ts', - { twoslash: true }, - // @ts-ignore Why shiki-twoslash requires a theme name? - {}, - highlighter, - twoslash - ); + html = highlight(source, 'ts', !/^(tutorial)/.test(filename)); } catch (e) { console.error(`Error compiling snippet in ${filename}`); // @ts-ignore @@ -1020,13 +990,20 @@ function syntax_highlight({ source, filename, language, highlighter, twoslashBan }) .join('')}`; } else { - const highlighted = highlighter.codeToHtml(source, { - lang: SHIKI_LANGUAGE_MAP[/** @type {keyof typeof SHIKI_LANGUAGE_MAP} */ (language)] - }); + const highlighted = highlight( + source, + SHIKI_LANGUAGE_MAP[/** @type {keyof typeof SHIKI_LANGUAGE_MAP} */ (language)] ?? language + ); html = replace_blank_lines(highlighted); } + // Remove any instance of element: + //
, it can have any content inside. + // And too + html = html.replace(/
[^]*?<\/div>/g, ''); + html = html.replace(/<\s*span\s*class="error-behind"[^>]*>.*?<\s*\/span\s*>/gs, ''); + return html; } diff --git a/packages/site-kit/src/lib/markdown/shiki.js b/packages/site-kit/src/lib/markdown/shiki.js new file mode 100644 index 00000000..338664a3 --- /dev/null +++ b/packages/site-kit/src/lib/markdown/shiki.js @@ -0,0 +1,63 @@ +import { rendererClassic, transformerTwoslash } from '@shikijs/twoslash'; +import { createCssVariablesTheme, getWasmInlined } from 'shiki'; +import { getHighlighterCore } from 'shiki/core'; +import css from 'shiki/langs/css.mjs'; +import diff from 'shiki/langs/diff.mjs'; +import html from 'shiki/langs/html.mjs'; +import javascript from 'shiki/langs/javascript.mjs'; +import json from 'shiki/langs/json.mjs'; +import markdown from 'shiki/langs/markdown.mjs'; +import bash from 'shiki/langs/shellscript.mjs'; +import svelte from 'shiki/langs/svelte.mjs'; +import typescript from 'shiki/langs/typescript.mjs'; +import yaml from 'shiki/langs/yaml.mjs'; +import ts from 'typescript'; + +const svelte_theme = createCssVariablesTheme({ + name: 'css-variables', + variablePrefix: '--shiki-', + variableDefaults: {}, + fontStyle: true +}); + +const shiki = await getHighlighterCore({ + themes: [svelte_theme], + langs: [javascript, svelte, typescript, diff, json, yaml, markdown, html, css, bash], + loadWasm: getWasmInlined, + langAlias: { + sh: 'bash' + } +}); + +/** + * @param {string} code + * @param {string} lang + * @param {boolean} twoslash + */ +export function highlight(code, lang, twoslash = false) { + return shiki.codeToHtml(code, { + lang, + themes: { light: 'css-variables' }, + theme: 'css-variables', + transformers: [ + transformerTwoslash({ + renderer: rendererClassic(), + filter: () => twoslash, + onTwoslashError: (error, code, lang) => { + console.error('Shiki twoslash error: ', { lang, code, error }); + }, + onShikiError: (error, code, lang) => { + console.error('Shiki error: ', { lang, code, error }); + }, + twoslashOptions: { + compilerOptions: { + allowJs: true, + checkJs: true, + target: ts.ScriptTarget.ES2022, + types: ['svelte', '@sveltejs/kit', '@types/node'] + } + } + }) + ] + }); +} diff --git a/packages/site-kit/src/lib/styles/code.css b/packages/site-kit/src/lib/styles/code.css index d9095bc8..2f89d416 100644 --- a/packages/site-kit/src/lib/styles/code.css +++ b/packages/site-kit/src/lib/styles/code.css @@ -4,7 +4,7 @@ body, body.light, body.dark { --shiki-color-text: var(--sk-code-base); - --shiki-color-background: hsl(var(--sk-back-3-hsl)); + --shiki-background: hsl(var(--sk-back-3-hsl)); --shiki-token-constant: var(--sk-code-base); --shiki-token-string: var(--sk-code-string); --shiki-token-comment: var(--sk-code-comment); diff --git a/packages/site-kit/src/lib/styles/text.css b/packages/site-kit/src/lib/styles/text.css index 8756bd56..973fc9a0 100644 --- a/packages/site-kit/src/lib/styles/text.css +++ b/packages/site-kit/src/lib/styles/text.css @@ -115,6 +115,16 @@ color: var(--sk-code-diff-base); } +.text :where(pre.shiki code) { + display: flex; + flex-direction: column; +} + +.text :where(pre.shiki code .line) { + flex-grow: 1 1 auto; + min-height: 22.09px; +} + .language-diff :where(.inserted, .deleted) { position: relative; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c605ec27..811539d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,6 +21,9 @@ importers: specifier: ^0.9.2 version: 0.9.2(svelte@4.2.16) devDependencies: + '@shikijs/twoslash': + specifier: ^1.5.1 + version: 1.5.1(typescript@5.4.5) '@sveltejs/kit': specifier: ^2.5.7 version: 2.5.7(@sveltejs/vite-plugin-svelte@3.1.0)(svelte@4.2.16)(vite@5.2.11) @@ -42,6 +45,9 @@ importers: prettier: specifier: ^3.2.5 version: 3.2.5 + shiki: + specifier: ^1.5.1 + version: 1.5.1 shiki-twoslash: specifier: ^3.1.2 version: 3.1.2(typescript@5.4.5) @@ -682,6 +688,20 @@ packages: dev: true optional: true + /@shikijs/core@1.5.1: + resolution: {integrity: sha512-xjV63pRUBvxA1LsxOUhRKLPh0uUjwBLzAKLdEuYSLIylo71sYuwDcttqNP01Ib1TZlLfO840CXHPlgUUsYFjzg==} + dev: true + + /@shikijs/twoslash@1.5.1(typescript@5.4.5): + resolution: {integrity: sha512-O0cnGcpW1LkBLd85TQp7Kdb9qzhSGyYl9c21BCAmYWhQdtnxaSKBgbiP3S35ewP/s3SrR9gCzumgznp/YSyMNg==} + dependencies: + '@shikijs/core': 1.5.1 + twoslash: 0.2.6(typescript@5.4.5) + transitivePeerDependencies: + - supports-color + - typescript + dev: true + /@sveltejs/kit@2.5.7(@sveltejs/vite-plugin-svelte@3.1.0)(svelte@4.2.16)(vite@5.2.11): resolution: {integrity: sha512-6uedTzrb7nQrw6HALxnPrPaXdIN2jJJTzTIl96Z3P5NiG+OAfpdPbrWrvkJ3GN4CfWqrmU4dJqwMMRMTD/C7ow==} engines: {node: '>=18.13'} @@ -817,6 +837,14 @@ packages: - supports-color dev: true + /@typescript/vfs@1.5.0: + resolution: {integrity: sha512-AJS307bPgbsZZ9ggCT3wwpg3VbTKMFNHfaY/uF0ahSkYYrPF2dSSKDNIDIQAHm9qJqbLvCsSJH7yN4Vs/CsMMg==} + dependencies: + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: true + /acorn@8.11.3: resolution: {integrity: sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==} engines: {node: '>=0.4.0'} @@ -2428,6 +2456,12 @@ packages: vscode-textmate: 5.2.0 dev: true + /shiki@1.5.1: + resolution: {integrity: sha512-vx4Ds3M3B9ZEmLeSXqBAB85osBWV8ErZfP69kuFQZozPgHc33m7spLTCUkcjwEjFm3gk3F9IdXMv8kX+v9xDHA==} + dependencies: + '@shikijs/core': 1.5.1 + dev: true + /side-channel@1.0.6: resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} engines: {node: '>= 0.4'} @@ -2689,6 +2723,22 @@ packages: yargs: 17.7.2 dev: true + /twoslash-protocol@0.2.6: + resolution: {integrity: sha512-8NbJlYeRdBcCTQ7ui7pdRPC1NL16aOnoYNz06oBW+W0qWNuiQXHgE8UeNvbA038aDd6ZPuuD5WedsBIESocB4g==} + dev: true + + /twoslash@0.2.6(typescript@5.4.5): + resolution: {integrity: sha512-DcAKIyXMB6xNs+SOw/oF8GvUr/cfJSqznngVXYbAUIVfTW3M8vWSEoCaz/RgSD+M6vwtK8DJ4/FmYBF5MN8BGw==} + peerDependencies: + typescript: '*' + dependencies: + '@typescript/vfs': 1.5.0 + twoslash-protocol: 0.2.6 + typescript: 5.4.5 + transitivePeerDependencies: + - supports-color + dev: true + /type-fest@0.13.1: resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} engines: {node: '>=10'} From 7b99d81acc2292010d4d93ac52a40b64cc81bae3 Mon Sep 17 00:00:00 2001 From: Puru Date: Mon, 20 May 2024 23:46:30 +0530 Subject: [PATCH 2/2] Push --- packages/site-kit/src/lib/markdown/shiki.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/site-kit/src/lib/markdown/shiki.js b/packages/site-kit/src/lib/markdown/shiki.js index 338664a3..b9a4e165 100644 --- a/packages/site-kit/src/lib/markdown/shiki.js +++ b/packages/site-kit/src/lib/markdown/shiki.js @@ -43,12 +43,12 @@ export function highlight(code, lang, twoslash = false) { transformerTwoslash({ renderer: rendererClassic(), filter: () => twoslash, - onTwoslashError: (error, code, lang) => { - console.error('Shiki twoslash error: ', { lang, code, error }); - }, - onShikiError: (error, code, lang) => { - console.error('Shiki error: ', { lang, code, error }); - }, + // onTwoslashError: (error, code, lang) => { + // console.error('Shiki twoslash error: ', { lang, code, error }); + // }, + // onShikiError: (error, code, lang) => { + // console.error('Shiki error: ', { lang, code, error }); + // }, twoslashOptions: { compilerOptions: { allowJs: true,