puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-11-08 17:34:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(highlight): handle Unicode heredoc delimiters and repeated template markers

Browse Source 
Heredoc delimiters only accepted ASCII, so a Unicode delimiter was recorded
as empty and the rest of the file rendered as a string. `$$${` consumed the
second marker before confirming `{` followed, which turned an escape into an
interpolation.
This commit is contained in:
Mohamed Boudra
2026-11-08 14:22:56 +01:00
parent b03695d787
commit 586b605429
2 changed files with 34 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/highlight/src/__tests__/highlighter.test.ts
+16
View File
@@ -401,6 +401,22 @@ describe("Terraform", () => {
expect(styleOf(result[3], "3")).toBe("number");
});
it("ends a heredoc with a Unicode delimiter", () => {
const result = highlightCode("x = <<ÉOF\nfoo\nÉOF\ny = 1", "main.tf");
expect(styleOf(result[3], "y")).toBe("definition");
expect(styleOf(result[3], "1")).toBe("number");
});
it.each([
['x = "$$${var.name}"', "$"],
['x = "%%%{ if true }"', "%"],
])("keeps escaped template text in %s as a string", (code) => {
const line = highlightCode(code, "main.tf")[0];
expect(line[line.length - 1]).toEqual({ text: code.slice(4), style: "string" });
});
it("parses a real configuration without error nodes", () => {
const code = [
"terraform {",
packages/highlight/src/hcl/tokens.ts
+18 -35
View File
@@ -164,6 +164,21 @@ function scanHeredocBody(input: InputStream, expected: string): void {
}
}
// `$${` and `%%{` are escapes. The doubled marker counts only when `{` follows it, so in `$$${`
// the first `$` is literal text and the rest is the escape.
function scanTemplateMarker(input: InputStream, marker: string, startToken: number): void {
input.advance();
if (String.fromCharCode(input.next) === "{") {
input.advance();
return input.acceptToken(startToken);
}
if (String.fromCharCode(input.next) === marker && String.fromCharCode(input.peek(1)) === "{") {
input.advance();
input.advance();
}
input.acceptToken(templateLiteralChunk);
}
export const scanQuotedTemplateStart = new ExternalTokenizer(
(input, stack) => {
if (
@@ -225,23 +240,7 @@ export const scanTemplateInterpolationStart = new ExternalTokenizer(
!(stack.context && stack.context.type === ContextType.TEMPLATE_INTERPOLATION) &&
String.fromCharCode(input.next) === "$"
) {
input.advance();
if (String.fromCharCode(input.next) === "{") {
input.advance();
return input.acceptToken(TemplateInterpolationStart);
}
// escape sequence
if (String.fromCharCode(input.next) === "$") {
input.advance();
if (String.fromCharCode(input.next) === "{") {
// $${
input.advance();
return input.acceptToken(templateLiteralChunk);
}
}
return input.acceptToken(templateLiteralChunk);
return scanTemplateMarker(input, "$", TemplateInterpolationStart);
}
},
{ contextual: true, fallback: true },
@@ -268,23 +267,7 @@ export const scanTemplateDirectiveStart = new ExternalTokenizer(
!(stack.context && stack.context.type === ContextType.TEMPLATE_DIRECTIVE) &&
String.fromCharCode(input.next) === "%"
) {
input.advance();
if (String.fromCharCode(input.next) === "{") {
input.advance();
return input.acceptToken(TemplateDirectiveStart);
}
// escape sequence
if (String.fromCharCode(input.next) === "%") {
input.advance();
if (String.fromCharCode(input.next) === "{") {
// $${
input.advance();
return input.acceptToken(templateLiteralChunk);
}
}
return input.acceptToken(templateLiteralChunk);
return scanTemplateMarker(input, "%", TemplateDirectiveStart);
}
},
{ contextual: true, fallback: true },
@@ -312,7 +295,7 @@ export const scanHeredocIdentifier = new ExternalTokenizer(
input.next !== -1
) {
let next = String.fromCharCode(input.next);
while (/[0-9a-zA-Z]/.test(next) || next === "_" || next === "-") {
while (/[\p{L}\p{N}_-]/u.test(next)) {
input.advance();
next = String.fromCharCode(input.next);
}