fix(editor): register comment tokens for Lezer languages

packages/app/e2e/browser/file-editing.spec.ts

@@ -96,6 +96,37 @@ async function seedAgentWithFileLink(input: LinkedFile) {
 }
 
 test.describe("CodeMirror workspace file editing", () => {
+  test("toggles comments using each file language's tokens", async ({ page, withWorkspace }) => {
+    const workspace = await withWorkspace({ prefix: "file-comment-shortcut-" });
+    const sources = [
+      { filename: "comment.ts", content: "const value = 1;", commented: "// const value = 1;" },
+      { filename: "comment.py", content: "value = 1", commented: "# value = 1" },
+      { filename: "comment.css", content: "body {}", commented: "/* body {} */" },
+    ];
+    for (const source of sources) {
+      await writeFile(path.join(workspace.repoPath, source.filename), source.content, "utf8");
+    }
+    await workspace.navigateTo();
+    for (const source of sources) {
+      await openWorkspaceFile(page, source.filename);
+      await editor(page).click();
+      await editor(page).press("ControlOrMeta+/");
+      await expect(editor(page)).toHaveText(source.commented);
+      await expect
+        .poll(() => readFile(path.join(workspace.repoPath, source.filename), "utf8"))
+        .toBe(source.commented);
+      await editor(page).press("ControlOrMeta+/");
+      await expect(editor(page)).toHaveText(source.content);
+      await expect
+        .poll(() => readFile(path.join(workspace.repoPath, source.filename), "utf8"))
+        .toBe(source.content);
+    }
+    await test.info().attach("comment-shortcut", {
+      body: await page.screenshot(),
+      contentType: "image/png",
+    });
+  });
+
   test("shows an absolute POSIX assistant file link relative to the workspace on hover", async ({
     page,
   }) => {

packages/app/src/file-pane/editor/language.test.ts

@@ -0,0 +1,45 @@
+import { EditorState, type Transaction } from "@codemirror/state";
+import { toggleComment } from "@codemirror/commands";
+import { getLanguageForFile } from "@getpaseo/highlight";
+import { expect, test } from "vitest";
+
+test.each([
+  ["source.ts", "const value = 1;", "// const value = 1;"],
+  ["source.py", "value = 1", "# value = 1"],
+  ["source.css", "body {}", "/* body {} */"],
+  ["source.html", "<p>text</p>", "<!-- <p>text</p> -->"],
+  ["source.md", "text", "<!-- text -->"],
+  ["source.ex", "value = 1", "# value = 1"],
+  ["source.cs", "var value = 1;", "// var value = 1;"],
+  ["source.nix", "1", "# 1"],
+])("toggles the comment and restores the source in %s", (filename, doc, commented) => {
+  const language = getLanguageForFile(filename);
+  if (!language) throw new Error(`Missing language for ${filename}`);
+  let state = EditorState.create({ doc, extensions: [language] });
+  const target = {
+    get state() {
+      return state;
+    },
+    dispatch(transaction: Transaction) {
+      state = transaction.state;
+    },
+  };
+  expect(toggleComment(target)).toBe(true);
+  expect(state.doc.toString()).toBe(commented);
+  expect(toggleComment(target)).toBe(true);
+  expect(state.doc.toString()).toBe(doc);
+});
+
+test("keeps JSON comment-free", () => {
+  const language = getLanguageForFile("data.json");
+  if (!language) throw new Error("Missing JSON language");
+  const state = EditorState.create({ doc: "{}", extensions: [language] });
+  expect(
+    toggleComment({
+      state,
+      dispatch() {
+        throw new Error("JSON must not be edited");
+      },
+    }),
+  ).toBe(false);
+});

packages/highlight/src/parsers.ts

@@ -1,4 +1,9 @@
-import { defineLanguageFacet, Language, StreamLanguage } from "@codemirror/language";
+import {
+  defineLanguageFacet,
+  Language,
+  languageDataProp,
+  StreamLanguage,
+} from "@codemirror/language";
 import { dart } from "@codemirror/legacy-modes/mode/clike";
 import { swift } from "@codemirror/legacy-modes/mode/swift";
 import { parser as jsParser } from "@lezer/javascript";
@@ -9,68 +14,86 @@ import { parser as goParser } from "@lezer/go";
 import { parser as htmlParser } from "@lezer/html";
 import { parser as javaParser } from "@lezer/java";
 import { parser as pythonParser } from "@lezer/python";
-import { parser as markdownParser } from "@lezer/markdown";
+import { parser as markdownParser, type MarkdownParser } from "@lezer/markdown";
 import { parser as phpParser } from "@lezer/php";
 import { parser as rustParser } from "@lezer/rust";
 import { parser as xmlParser } from "@lezer/xml";
 import { parser as yamlParser } from "@lezer/yaml";
 import { parser as elixirParser } from "lezer-elixir";
 import type { Parser } from "@lezer/common";
+import type { LRParser } from "@lezer/lr";
 import { csharpLanguage } from "./csharp/language.js";
 import { astroParser } from "./astro/parser.js";
 import { nixLanguage } from "./nix/language.js";
 import { parser as svelteBaseParser } from "./svelte/parser.js";
 import { configureNesting, defaultNesting } from "./svelte/nesting.js";
 
-function language(parser: Parser): Language {
-  return new Language(defineLanguageFacet(), parser);
+const cStyleComments = { commentTokens: { line: "//", block: { open: "/*", close: "*/" } } };
+const hashComments = { commentTokens: { line: "#" } };
+const cssComments = { commentTokens: { block: { open: "/*", close: "*/" } } };
+const markupComments = { commentTokens: { block: { open: "<!--", close: "-->" } } };
+
+function language(
+  parser: LRParser | MarkdownParser,
+  languageData: Parameters<typeof defineLanguageFacet>[0] = {},
+): Language {
+  const data = defineLanguageFacet(languageData);
+  return new Language(
+    data,
+    parser.configure({
+      props: [languageDataProp.add((node) => (node.isTop ? data : undefined))],
+    }),
+  );
 }
 
 const languagesByExtension: Record<string, Language> = {
   // JavaScript/TypeScript
-  js: language(jsParser),
-  jsx: language(jsParser.configure({ dialect: "jsx" })),
-  ts: language(jsParser.configure({ dialect: "ts" })),
-  tsx: language(jsParser.configure({ dialect: "ts jsx" })),
-  mjs: language(jsParser),
-  cjs: language(jsParser),
+  js: language(jsParser, cStyleComments),
+  jsx: language(jsParser.configure({ dialect: "jsx" }), cStyleComments),
+  ts: language(jsParser.configure({ dialect: "ts" }), cStyleComments),
+  tsx: language(jsParser.configure({ dialect: "ts jsx" }), cStyleComments),
+  mjs: language(jsParser, cStyleComments),
+  cjs: language(jsParser, cStyleComments),
   // C / C++ / Objective-C
-  c: language(cppParser),
-  h: language(cppParser),
-  cc: language(cppParser),
-  cpp: language(cppParser),
-  cxx: language(cppParser),
-  hpp: language(cppParser),
-  hxx: language(cppParser),
-  m: language(cppParser),
-  mm: language(cppParser),
+  c: language(cppParser, cStyleComments),
+  h: language(cppParser, cStyleComments),
+  cc: language(cppParser, cStyleComments),
+  cpp: language(cppParser, cStyleComments),
+  cxx: language(cppParser, cStyleComments),
+  hpp: language(cppParser, cStyleComments),
+  hxx: language(cppParser, cStyleComments),
+  m: language(cppParser, cStyleComments),
+  mm: language(cppParser, cStyleComments),
   // JSON
   json: language(jsonParser),
   // CSS
-  css: language(cssParser),
-  scss: language(cssParser),
+  css: language(cssParser, cssComments),
+  scss: language(cssParser, cStyleComments),
   // HTML
-  html: language(htmlParser),
-  htm: language(htmlParser),
+  html: language(htmlParser, markupComments),
+  htm: language(htmlParser, markupComments),
   // Svelte
-  svelte: language(svelteBaseParser.configure({ wrap: configureNesting(defaultNesting) })),
+  svelte: language(
+    svelteBaseParser.configure({ wrap: configureNesting(defaultNesting) }),
+    markupComments,
+  ),
   // Astro
-  astro: language(astroParser),
+  astro: new Language(defineLanguageFacet(), astroParser),
   // XML
-  xml: language(xmlParser),
+  xml: language(xmlParser, markupComments),
   // Java
-  java: language(javaParser),
+  java: language(javaParser, cStyleComments),
   // Python
-  py: language(pythonParser),
+  py: language(pythonParser, hashComments),
   // Go
-  go: language(goParser),
+  go: language(goParser, cStyleComments),
   // PHP
-  php: language(phpParser),
+  php: language(phpParser, cStyleComments),
   // YAML
-  yaml: language(yamlParser),
-  yml: language(yamlParser),
+  yaml: language(yamlParser, hashComments),
+  yml: language(yamlParser, hashComments),
   // Rust
-  rs: language(rustParser),
+  rs: language(rustParser, cStyleComments),
   // Swift
   swift: StreamLanguage.define(swift),
   // Dart
@@ -80,11 +103,11 @@ const languagesByExtension: Record<string, Language> = {
   // Nix
   nix: nixLanguage,
   // Elixir
-  ex: language(elixirParser),
-  exs: language(elixirParser),
+  ex: language(elixirParser, hashComments),
+  exs: language(elixirParser, hashComments),
   // Markdown
-  md: language(markdownParser),
-  mdx: language(markdownParser),
+  md: language(markdownParser, markupComments),
+  mdx: language(markdownParser, markupComments),
 };
 
 export function getLanguageForFile(filename: string): Language | null {