Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions src/haxeparser/HaxeLexer.hx
Original file line number Diff line number Diff line change
Expand Up @@ -360,9 +360,20 @@ class HaxeLexer extends Lexer implements hxparse.RuleBuilder {

var depth = 0;
var index = 0;
// The cursor only advances. Retain absent matches and rescan a found
// delimiter only after passing it, avoiding repeated suffix scans.
var nextEndTag = text.indexOf(endTag, index);
var nextSelfClosing = text.indexOf("/>", index);
// Without either closing form, no opening tag can return depth to zero.
if (nextEndTag == -1 && nextSelfClosing == -1) {
return normalLt();
}
while (true) {
var indexStartTag = text.indexOf(startTag, index);
var indexEndTag = text.indexOf(endTag, index);
if (nextEndTag >= 0 && nextEndTag < index) {
nextEndTag = text.indexOf(endTag, index);
}
var indexEndTag = nextEndTag;
if ((indexStartTag == -1) && (indexEndTag == -1)) {
return normalLt();
}
Expand All @@ -381,7 +392,10 @@ class HaxeLexer extends Lexer implements hxparse.RuleBuilder {
continue;
}
depth++;
var indexSelfClosing = text.indexOf("/>", index);
if (nextSelfClosing >= 0 && nextSelfClosing < index) {
nextSelfClosing = text.indexOf("/>", index);
}
var indexSelfClosing = nextSelfClosing;
var indexTagClosing = text.indexOf(">", index);
var indexOpenTag = text.indexOf("<", index);

Expand Down
1 change: 1 addition & 0 deletions test/TestMain.hx
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class TestMain {
{
var tests : Array<ITest> = [
new Test (),
new TestMarkupScan (),
#if haxe_std_path new TestStd() #end
];

Expand Down
43 changes: 43 additions & 0 deletions test/TestMarkupScan.hx
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import byte.ByteData;
import haxeparser.HaxeLexer;
import utest.Assert;

/** Checks that delimiter search reuse preserves tokens and source byte offsets. */
class TestMarkupScan implements utest.ITest {
public function new() {}

function testNestedAndAdjacentMarkup() {
for (markup in ["<a/>", "<a><a/></a>", "<a><a></a></a>", "<a><ab/></a>"]) {
var lexer = new HaxeLexer(ByteData.ofString(markup + " + 1"), "markup.hx");
var token = lexer.token(HaxeLexer.tok);
Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup(markup)), token.tok);
Assert.equals(0, token.pos.min);
Assert.equals(ByteData.ofString(markup).length, token.pos.max);
Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpAdd), lexer.token(HaxeLexer.tok).tok);
}
var lexer = new HaxeLexer(ByteData.ofString("<a/><a></a>"), "markup.hx");
Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup("<a/>")), lexer.token(HaxeLexer.tok).tok);
Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup("<a></a>")), lexer.token(HaxeLexer.tok).tok);
}

function testUnicodeMarkupResumesAtNextToken() {
var markup = "<a>é😀</a>";
var lexer = new HaxeLexer(ByteData.ofString(markup + " + 1"), "markup.hx");
var token = lexer.token(HaxeLexer.tok);
Assert.same(haxeparser.Data.TokenDef.Const(haxeparser.Data.Constant.CMarkup(markup)), token.tok);
Assert.equals(ByteData.ofString(markup).length, token.pos.max);
var following = lexer.token(HaxeLexer.tok);
Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpAdd), following.tok);
Assert.equals(ByteData.ofString(markup).length + 1, following.pos.min);
}

function testIncompleteMarkupReturnsLessThan() {
for (source in ["<a", "<a>", "<a><a>", "<a></b>", "<Int>", "<a </aSuffix>"]) {
var lexer = new HaxeLexer(ByteData.ofString(source), "markup.hx");
var token = lexer.token(HaxeLexer.tok);
Assert.same(haxeparser.Data.TokenDef.Binop(haxe.macro.Expr.Binop.OpLt), token.tok);
Assert.equals(0, token.pos.min);
Assert.equals(1, token.pos.max);
}
}
}