@expressive-code/plugin-shiki - npm Package Compare versions

Comparing version 0.37.1 to 0.38.0

dist/index.d.ts

 import { ExpressiveCodeTheme, ExpressiveCodePlugin } from '@expressive-code/core';
 import { MaybeGetter, MaybeArray, LanguageRegistration as LanguageRegistration$1, ShikiTransformer, bundledThemes } from 'shiki';
 
-type Optional<T, K extends keyof T> = Omit<T, K> & Pick<Partial<T>, K>;
-type IRawRepository = Optional<LanguageRegistration$1['repository'], '$self' | '$base'>;
+type IRawRepository = LanguageRegistration$1['repository'];
 interface LanguageRegistration extends Omit<LanguageRegistration$1, 'repository'> {
@@ -23,2 +22,22 @@ repository?: IRawRepository | undefined;
     /**
+     * By default, the additional languages defined in `langs` are only available in
+     * top-level code blocks contained directly in their parent Markdown or MDX document.
+     *
+     * Setting this option to `true` also enables syntax highlighting when a fenced code block
+     * using one of your additional `langs` is nested inside an outer `markdown`, `md` or `mdx`
+     * code block. Example:
+     *
+     * `````md
+     * ````md
+     * This top-level Markdown code block contains a nested `my-custom-lang` code block:
+     *
+     * ```my-custom-lang
+     * This nested code block will only be highlighted using `my-custom-lang`
+     * if `injectLangsIntoNestedCodeBlocks` is enabled.
+     * ```
+     * ````
+     * `````
+     */
+    injectLangsIntoNestedCodeBlocks?: boolean | undefined;
+    /**
      * An optional list of Shiki transformers.
@@ -25,0 +44,0 @@ *

dist/index.js

@@ -8,3 +8,2 @@ // src/index.ts
 var highlighterPromiseByConfig = /* @__PURE__ */ new Map();
-var promisesByHighlighter = /* @__PURE__ */ new WeakMap();
 var themeCacheKeysByStyleVariants = /* @__PURE__ */ new WeakMap();
@@ -15,11 +14,11 @@ async function getCachedHighlighter(config = {}) {
   if (highlighterPromise === void 0) {
-    const langs = [];
-    if (config.langs?.length) {
-      langs.push(...config.langs);
-    }
-    langs.push(...Object.keys(bundledLanguages));
-    highlighterPromise = createHighlighter({
-      themes: [],
-      langs
-    });
+    highlighterPromise = (async () => {
+      const highlighter = await createHighlighter({
+        themes: [],
+        langs: []
+      });
+      if (config.langs?.length)
+        await ensureLanguagesAreLoaded(highlighter, config.langs, config.injectLangsIntoNestedCodeBlocks);
+      return highlighter;
+    })();
     highlighterPromiseByConfig.set(configCacheKey, highlighterPromise);
@@ -39,39 +38,196 @@ }
     themeCacheKeys.set(theme, cacheKey);
-  if (!highlighter.getLoadedThemes().includes(cacheKey)) {
-    await memoizeHighlighterTask(highlighter, `loadTheme:${cacheKey}`, () => {
-      const themeUsingCacheKey = { ...theme, name: cacheKey, settings: theme.settings ?? [] };
-      return highlighter.loadTheme(themeUsingCacheKey);
-    });
-  }
+  await runHighlighterTask(async () => {
+    if (highlighter.getLoadedThemes().includes(cacheKey))
+      return;
+    const themeUsingCacheKey = { ...theme, name: cacheKey, settings: theme.settings ?? [] };
+    await highlighter.loadTheme(themeUsingCacheKey);
+  });
   return cacheKey;
 }
-async function ensureLanguageIsLoaded(highlighter, language) {
-  const loadedLanguages = new Set(highlighter.getLoadedLanguages());
-  const isLoaded = loadedLanguages.has(language);
-  const isSpecial = isSpecialLang(language);
-  const isBundled = Object.keys(bundledLanguages).includes(language);
-  const isAvailable = isLoaded || isSpecial || isBundled;
-  if (!isAvailable)
-    return "txt";
-  if (isLoaded || isSpecial)
-    return language;
-  const loadedLanguage = await memoizeHighlighterTask(highlighter, `loadLanguage:${language}`, async () => {
-    await highlighter.loadLanguage(language);
-    return language;
+async function ensureLanguagesAreLoaded(highlighter, languages, injectLangsIntoNestedCodeBlocks = false) {
+  const errors = [];
+  await runHighlighterTask(async () => {
+    const loadedLanguages = new Set(highlighter.getLoadedLanguages());
+    const handledLanguageNames = /* @__PURE__ */ new Set();
+    const registrations = /* @__PURE__ */ new Map();
+    async function resolveLanguage(language, referencedBy = "") {
+      let languageInput;
+      if (typeof language === "string") {
+        if (handledLanguageNames.has(language))
+          return [];
+        handledLanguageNames.add(language);
+        if (loadedLanguages.has(language) || isSpecialLang(language))
+          return [];
+        if (!Object.keys(bundledLanguages).includes(language)) {
+          errors.push(`Unknown language "${language}"${referencedBy ? `, referenced by language(s): ${referencedBy}` : ""}`);
+          return [];
+        }
+        languageInput = bundledLanguages[language];
+      } else {
+        languageInput = language;
+      }
+      const potentialModule = await Promise.resolve(typeof languageInput === "function" ? languageInput() : languageInput);
+      const potentialArray = "default" in potentialModule ? potentialModule.default : potentialModule;
+      const languageRegistrations = Array.isArray(potentialArray) ? potentialArray : [potentialArray];
+      languageRegistrations.forEach((lang) => {
+        if (loadedLanguages.has(lang.name))
+          return;
+        const registration = { repository: {}, ...lang, embeddedLangsLazy: [] };
+        registrations.set(lang.name, registration);
+      });
+      if (injectLangsIntoNestedCodeBlocks && !referencedBy) {
+        languageRegistrations.forEach((lang) => {
+          const injectionLangs = getNestedCodeBlockInjectionLangs(lang);
+          injectionLangs.forEach((injectionLang) => registrations.set(injectionLang.name, injectionLang));
+        });
+      }
+      const referencedLangs = [...new Set(languageRegistrations.map((lang) => lang.embeddedLangsLazy ?? []).flat())];
+      const referencers = languageRegistrations.map((lang) => lang.name).join(", ");
+      await Promise.all(referencedLangs.map((lang) => resolveLanguage(lang, referencers)));
+    }
+    await Promise.all(languages.map((lang) => resolveLanguage(lang)));
+    if (registrations.size)
+      await highlighter.loadLanguage(...registrations.values());
   });
-  return loadedLanguage;
+  return errors;
 }
-function memoizeHighlighterTask(highlighter, taskId, taskFn) {
-  let promises = promisesByHighlighter.get(highlighter);
-  if (!promises) {
-    promises = /* @__PURE__ */ new Map();
-    promisesByHighlighter.set(highlighter, promises);
+var taskQueue = [];
+var processingQueue = false;
+function runHighlighterTask(taskFn) {
+  return new Promise((resolve, reject) => {
+    taskQueue.push({ taskFn, resolve, reject });
+    if (!processingQueue) {
+      processingQueue = true;
+      processQueue().catch((error) => {
+        processingQueue = false;
+        console.error("Error in Shiki highlighter task queue:", error);
+      });
+    }
+  });
+}
+async function processQueue() {
+  try {
+    while (taskQueue.length > 0) {
+      const task = taskQueue.shift();
+      if (!task)
+        break;
+      const { taskFn, resolve, reject } = task;
+      try {
+        await taskFn();
+        resolve();
+      } catch (error) {
+        reject(error);
+      }
+    }
+  } finally {
+    processingQueue = false;
   }
-  let promise = promises.get(taskId);
-  if (promise === void 0) {
-    promise = taskFn();
-    promises.set(taskId, promise);
-  }
-  return promise;
 }
+function getNestedCodeBlockInjectionLangs(lang) {
+  const injectionLangs = [];
+  const langNameKey = lang.name.replace(/[^a-zA-Z0-9]/g, "_");
+  const langNameAndAliases = [lang.name, ...lang.aliases ?? []];
+  injectionLangs.push({
+    name: `${lang.name}-fenced-md`,
+    scopeName: `source.${lang.name}.fenced_code_block`,
+    injectTo: ["text.html.markdown"],
+    injectionSelector: "L:text.html.markdown",
+    patterns: [
+      {
+        include: `#fenced_code_block_${langNameKey}`
+      }
+    ],
+    repository: {
+      [`fenced_code_block_${langNameKey}`]: {
+        begin: `(^|\\G)(\\s*)(\`{3,}|~{3,})\\s*(?i:(${langNameAndAliases.join("|")})((\\s+|:|,|\\{|\\?)[^\`]*)?$)`,
+        beginCaptures: {
+          3: {
+            name: "punctuation.definition.markdown"
+          },
+          4: {
+            name: "fenced_code.block.language.markdown"
+          },
+          5: {
+            name: "fenced_code.block.language.attributes.markdown"
+          }
+        },
+        end: "(^|\\G)(\\2|\\s{0,3})(\\3)\\s*$",
+        endCaptures: {
+          3: {
+            name: "punctuation.definition.markdown"
+          }
+        },
+        name: "markup.fenced_code.block.markdown",
+        patterns: [
+          {
+            begin: "(^|\\G)(\\s*)(.*)",
+            while: "(^|\\G)(?!\\s*([`~]{3,})\\s*$)",
+            contentName: `meta.embedded.block.${lang.name}`,
+            patterns: [
+              {
+                include: lang.scopeName
+              }
+            ]
+          }
+        ]
+      }
+    }
+  });
+  injectionLangs.push({
+    name: `${lang.name}-fenced-mdx`,
+    scopeName: `source.${lang.name}.fenced_code_block`,
+    injectTo: ["source.mdx"],
+    injectionSelector: "L:source.mdx",
+    patterns: [
+      {
+        include: `#fenced_code_block_${langNameKey}`
+      }
+    ],
+    repository: {
+      [`fenced_code_block_${langNameKey}`]: {
+        begin: `(?:^|\\G)[\\t ]*(\`{3,})(?:[\\t ]*((?i:(?:.*\\.)?${langNameAndAliases.join("|")}))(?:[\\t ]+((?:[^\\n\\r\`])+))?)(?:[\\t ]*$)`,
+        beginCaptures: {
+          1: {
+            name: "string.other.begin.code.fenced.mdx"
+          },
+          2: {
+            name: "entity.name.function.mdx",
+            patterns: [
+              {
+                include: "#markdown-string"
+              }
+            ]
+          },
+          3: {
+            patterns: [
+              {
+                include: "#markdown-string"
+              }
+            ]
+          }
+        },
+        end: "(?:^|\\G)[\\t ]*(\\1)(?:[\\t ]*$)",
+        endCaptures: {
+          1: {
+            name: "string.other.end.code.fenced.mdx"
+          }
+        },
+        name: `markup.code.${lang.name}.mdx`,
+        patterns: [
+          {
+            begin: "(^|\\G)(\\s*)(.*)",
+            contentName: `meta.embedded.${lang.name}`,
+            patterns: [
+              {
+                include: lang.scopeName
+              }
+            ],
+            while: "(^|\\G)(?![\\t ]*([`~]{3,})[\\t ]*$)"
+          }
+        ]
+      }
+    }
+  });
+  return injectionLangs;
+}
 
@@ -176,3 +332,3 @@ // src/transformers.ts
 function pluginShiki(options = {}) {
-  const { langs } = options;
+  const { langs, injectLangsIntoNestedCodeBlocks } = options;
   validateTransformers(options);
@@ -190,3 +346,3 @@ return {
         try {
-          highlighter = await getCachedHighlighter({ langs });
+          highlighter = await getCachedHighlighter({ langs, injectLangsIntoNestedCodeBlocks });
         } catch (err) {
@@ -199,6 +355,7 @@ const error = err instanceof Error ? err : new Error(String(err));
         }
-        const loadedLanguageName = await ensureLanguageIsLoaded(highlighter, codeBlock.language);
+        const languageLoadErrors = await ensureLanguagesAreLoaded(highlighter, [codeBlock.language]);
+        const loadedLanguageName = languageLoadErrors.length ? "txt" : codeBlock.language;
         if (loadedLanguageName !== codeBlock.language) {
           logger.warn(
-            `Found unknown code block language "${codeBlock.language}" in ${codeBlock.parentDocument?.sourceFilePath ? `document "${codeBlock.parentDocument?.sourceFilePath}"` : "markdown/MDX document"}. Using "${loadedLanguageName}" instead. You can add custom languages using the "langs" config option.`
+            `Error while loading code block language "${codeBlock.language}" in ${codeBlock.parentDocument?.sourceFilePath ? `document "${codeBlock.parentDocument?.sourceFilePath}"` : "markdown/MDX document"}. Using "${loadedLanguageName}" instead. You can add custom languages using the "langs" config option. Error details: ${languageLoadErrors.join(", ")}`
           );
@@ -209,3 +366,3 @@ }
           const loadedThemeName = await ensureThemeIsLoaded(highlighter, theme, styleVariants);
-          let tokenLines;
+          let tokenLines = [];
           try {
@@ -218,7 +375,10 @@ const codeToTokensOptions = {
             runPreprocessHook({ options, code, codeBlock, codeToTokensOptions });
-            tokenLines = highlighter.codeToTokensBase(
-              code,
-              // @ts-expect-error: We took care that the language and theme are loaded
-              codeToTokensOptions
-            );
+            const codeToTokensBase = highlighter.codeToTokensBase;
+            await runHighlighterTask(() => {
+              tokenLines = codeToTokensBase(
+                code,
+                // @ts-expect-error: We took care that the language and theme are loaded
+                codeToTokensOptions
+              );
+            });
             tokenLines = runTokensHook({ options, code, codeBlock, codeToTokensOptions, tokenLines });
@@ -225,0 +385,0 @@ } catch (err) {

package.json

 {
   "name": "@expressive-code/plugin-shiki",
-  "version": "0.37.1",
+  "version": "0.38.0",
   "description": "Shiki syntax highlighting plugin for Expressive Code, a text marking & annotation engine for presenting source code on the web.",
@@ -25,7 +25,7 @@ "keywords": [],
   "dependencies": {
-    "@expressive-code/core": "^0.37.1",
-    "shiki": "^1.14.1"
+    "@expressive-code/core": "^0.38.0",
+    "shiki": "^1.22.2"
   },
   "devDependencies": {
-    "@internal/test-utils": "^0.2.30"
+    "@internal/test-utils": "^0.2.31"
   },
@@ -32,0 +32,0 @@ "scripts": {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

60374

increased by37.03%

Lines of code

510

increased by54.08%

Dependency changes

• + Added@expressive-code/core@0.38.3(transitive)

• - Removed@expressive-code/core@0.37.1(transitive)

• Updated@expressive-code/core@^0.38.0

• Updatedshiki@^1.22.2