diff --git a/src/haxeparser/HaxeLexer.hx b/src/haxeparser/HaxeLexer.hx index ca914d9..9dc5341 100644 --- a/src/haxeparser/HaxeLexer.hx +++ b/src/haxeparser/HaxeLexer.hx @@ -360,9 +360,20 @@ class HaxeLexer extends Lexer implements hxparse.RuleBuilder { var depth = 0; var index = 0; + // The cursor only advances. Retain absent matches and rescan a found + // delimiter only after passing it, avoiding repeated suffix scans. + var nextEndTag = text.indexOf(endTag, index); + var nextSelfClosing = text.indexOf("/>", index); + // Without either closing form, no opening tag can return depth to zero. + if (nextEndTag == -1 && nextSelfClosing == -1) { + return normalLt(); + } while (true) { var indexStartTag = text.indexOf(startTag, index); - var indexEndTag = text.indexOf(endTag, index); + if (nextEndTag >= 0 && nextEndTag < index) { + nextEndTag = text.indexOf(endTag, index); + } + var indexEndTag = nextEndTag; if ((indexStartTag == -1) && (indexEndTag == -1)) { return normalLt(); } @@ -381,7 +392,10 @@ class HaxeLexer extends Lexer implements hxparse.RuleBuilder { continue; } depth++; - var indexSelfClosing = text.indexOf("/>", index); + if (nextSelfClosing >= 0 && nextSelfClosing < index) { + nextSelfClosing = text.indexOf("/>", index); + } + var indexSelfClosing = nextSelfClosing; var indexTagClosing = text.indexOf(">", index); var indexOpenTag = text.indexOf("<", index); diff --git a/test/TestMain.hx b/test/TestMain.hx index 3f46629..1408429 100644 --- a/test/TestMain.hx +++ b/test/TestMain.hx @@ -10,6 +10,7 @@ class TestMain { { var tests : Array = [ new Test (), + new TestMarkupScan (), #if haxe_std_path new TestStd() #end ]; diff --git a/test/TestMarkupScan.hx b/test/TestMarkupScan.hx new file mode 100644 index 0000000..b3a2904 --- /dev/null +++ b/test/TestMarkupScan.hx @@ -0,0 +1,43 @@ +import byte.ByteData; +import haxeparser.HaxeLexer; +import utest.Assert; + +/** Checks that delimiter search reuse preserves tokens and source byte offsets. */ +class TestMarkupScan implements utest.ITest { + public function new() {} + + function testNestedAndAdjacentMarkup() { + for (markup in ["", "", "", ""]) { + var lexer = new HaxeLexer(ByteData.ofString(markup + " + 1"), "markup.hx"); + var token = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup(markup)), token.tok); + Assert.equals(0, token.pos.min); + Assert.equals(ByteData.ofString(markup).length, token.pos.max); + Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpAdd), lexer.token(HaxeLexer.tok).tok); + } + var lexer = new HaxeLexer(ByteData.ofString(""), "markup.hx"); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup("")), lexer.token(HaxeLexer.tok).tok); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup("")), lexer.token(HaxeLexer.tok).tok); + } + + function testUnicodeMarkupResumesAtNextToken() { + var markup = "é😀"; + var lexer = new HaxeLexer(ByteData.ofString(markup + " + 1"), "markup.hx"); + var token = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup(markup)), token.tok); + Assert.equals(ByteData.ofString(markup).length, token.pos.max); + var following = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpAdd), following.tok); + Assert.equals(ByteData.ofString(markup).length + 1, following.pos.min); + } + + function testIncompleteMarkupReturnsLessThan() { + for (source in ["", "", "", "", ""]) { + var lexer = new HaxeLexer(ByteData.ofString(source), "markup.hx"); + var token = lexer.token(HaxeLexer.tok); + Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpLt), token.tok); + Assert.equals(0, token.pos.min); + Assert.equals(1, token.pos.max); + } + } +}