From 9949dcc5af3aa7b66f923c8f0545af3d607345ae Mon Sep 17 00:00:00 2001 From: Marcelo Serpa Date: Wed, 9 Sep 2026 01:26:48 -0600 Subject: [PATCH] perf: avoid repeated inline markup closing-marker scans Large files with repeated generic types can make markup detection scan the remaining source repeatedly. Remember the next closing markers while the cursor advances, and return the less-than token when neither closing form exists. Keep this state local to one markup scan. Preserve the existing nesting decisions, token contents, and cursor updates. Add focused tests for nested and adjacent markup, incomplete input, and Unicode text followed by another token. Validation includes 5,199 differential lexer cases, the formatter's 1,831 assertions, and the current parser's 515 assertions on Haxe 4.3.7. The optimization does not repair the existing Unicode markup start offset or change formatter rules. Full consumer adoption remains separate. --- src/haxeparser/HaxeLexer.hx | 18 ++++++++++++++-- test/TestMain.hx | 1 + test/TestMarkupScan.hx | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 test/TestMarkupScan.hx diff --git a/src/haxeparser/HaxeLexer.hx b/src/haxeparser/HaxeLexer.hx index ca914d9..9dc5341 100644 --- a/src/haxeparser/HaxeLexer.hx +++ b/src/haxeparser/HaxeLexer.hx @@ -360,9 +360,20 @@ class HaxeLexer extends Lexer implements hxparse.RuleBuilder { var depth = 0; var index = 0; + // The cursor only advances. Retain absent matches and rescan a found + // delimiter only after passing it, avoiding repeated suffix scans. + var nextEndTag = text.indexOf(endTag, index); + var nextSelfClosing = text.indexOf("/>", index); + // Without either closing form, no opening tag can return depth to zero. + if (nextEndTag == -1 && nextSelfClosing == -1) { + return normalLt(); + } while (true) { var indexStartTag = text.indexOf(startTag, index); - var indexEndTag = text.indexOf(endTag, index); + if (nextEndTag >= 0 && nextEndTag < index) { + nextEndTag = text.indexOf(endTag, index); + } + var indexEndTag = nextEndTag; if ((indexStartTag == -1) && (indexEndTag == -1)) { return normalLt(); } @@ -381,7 +392,10 @@ class HaxeLexer extends Lexer implements hxparse.RuleBuilder { continue; } depth++; - var indexSelfClosing = text.indexOf("/>", index); + if (nextSelfClosing >= 0 && nextSelfClosing < index) { + nextSelfClosing = text.indexOf("/>", index); + } + var indexSelfClosing = nextSelfClosing; var indexTagClosing = text.indexOf(">", index); var indexOpenTag = text.indexOf("<", index); diff --git a/test/TestMain.hx b/test/TestMain.hx index 3f46629..1408429 100644 --- a/test/TestMain.hx +++ b/test/TestMain.hx @@ -10,6 +10,7 @@ class TestMain { { var tests : Array = [ new Test (), + new TestMarkupScan (), #if haxe_std_path new TestStd() #end ]; diff --git a/test/TestMarkupScan.hx b/test/TestMarkupScan.hx new file mode 100644 index 0000000..b3a2904 --- /dev/null +++ b/test/TestMarkupScan.hx @@ -0,0 +1,43 @@ +import byte.ByteData; +import haxeparser.HaxeLexer; +import utest.Assert; + +/** Checks that delimiter search reuse preserves tokens and source byte offsets. */ +class TestMarkupScan implements utest.ITest { + public function new() {} + + function testNestedAndAdjacentMarkup() { + for (markup in ["", "", "", ""]) { + var lexer = new HaxeLexer(ByteData.ofString(markup + " + 1"), "markup.hx"); + var token = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup(markup)), token.tok); + Assert.equals(0, token.pos.min); + Assert.equals(ByteData.ofString(markup).length, token.pos.max); + Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpAdd), lexer.token(HaxeLexer.tok).tok); + } + var lexer = new HaxeLexer(ByteData.ofString(""), "markup.hx"); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup("")), lexer.token(HaxeLexer.tok).tok); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup("")), lexer.token(HaxeLexer.tok).tok); + } + + function testUnicodeMarkupResumesAtNextToken() { + var markup = "é😀"; + var lexer = new HaxeLexer(ByteData.ofString(markup + " + 1"), "markup.hx"); + var token = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup(markup)), token.tok); + Assert.equals(ByteData.ofString(markup).length, token.pos.max); + var following = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpAdd), following.tok); + Assert.equals(ByteData.ofString(markup).length + 1, following.pos.min); + } + + function testIncompleteMarkupReturnsLessThan() { + for (source in ["", "", "", "", ""]) { + var lexer = new HaxeLexer(ByteData.ofString(source), "markup.hx"); + var token = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpLt), token.tok); + Assert.equals(0, token.pos.min); + Assert.equals(1, token.pos.max); + } + } +}