From: Alexandre Terrasa Date: Sat, 6 Jun 2015 22:04:11 +0000 (-0400) Subject: markdown: fix token location in multilines input X-Git-Tag: v0.7.7~40^2~5 X-Git-Url: http://nitlanguage.org markdown: fix token location in multilines input Signed-off-by: Alexandre Terrasa --- diff --git a/lib/markdown/markdown.nit b/lib/markdown/markdown.nit index eac11c2..854eed5 100644 --- a/lib/markdown/markdown.nit +++ b/lib/markdown/markdown.nit @@ -396,7 +396,11 @@ class MarkdownProcessor c2 = ' ' end - var loc = text.pos_to_loc(pos) + var loc = new MDLocation( + current_loc.line_start, + current_loc.column_start + pos, + current_loc.line_start, + current_loc.column_start + pos) if c == '*' then if c1 == '*' then @@ -476,6 +480,12 @@ class MarkdownProcessor end return -1 end + + # Location used for next parsed token. + # + # This location can be changed by the emitter to adjust with `\n` found + # in the input. + private fun current_loc: MDLocation do return emitter.current_loc end # Emit output corresponding to blocks content. @@ -519,15 +529,19 @@ class MarkdownEmitter # Transform and emit mardown text fun emit_text(text: Text) do emit_text_until(text, 0, null) - # Transform and emit mardown text starting at `from` and + # Transform and emit mardown text starting at `start` and # until a token with the same type as `token` is found. - # Go until the end of text if `token` is null. + # Go until the end of `text` if `token` is null. fun emit_text_until(text: Text, start: Int, token: nullable Token): Int do var old_text = current_text var old_pos = current_pos current_text = text current_pos = start while current_pos < text.length do + if text[current_pos] == '\n' then + current_loc.line_start += 1 + current_loc.column_start = -current_pos + end var mt = processor.token_at(text, current_pos) if (token != null and not token isa TokenNone) and (mt.is_same_type(token) or @@ -570,6 +584,21 @@ class MarkdownEmitter return buffer_stack.last end + # Stacked locations. + private var loc_stack = new List[MDLocation] + + # Push a new MDLocation on the stack. + private fun push_loc(location: MDLocation) do loc_stack.add location + + # Pop the last buffer. + private fun pop_loc: MDLocation do return loc_stack.pop + + # Current output buffer. + private fun current_loc: MDLocation do + assert not loc_stack.is_empty + return loc_stack.last + end + # Append `e` to current buffer. fun add(e: Writable) do if e isa Text then @@ -900,6 +929,11 @@ class MDLocation var column_end: Int redef fun to_s do return "{line_start},{column_start}--{line_end},{column_end}" + + # Return a copy of `self`. + fun copy: MDLocation do + return new MDLocation(line_start, column_start, line_end, column_end) + end end # A block of markdown lines. @@ -1128,7 +1162,9 @@ abstract class Block fun emit_blocks(v: MarkdownEmitter) do var block = self.block.first_block while block != null do + v.push_loc(block.location) block.kind.emit(v) + v.pop_loc block = block.next end end @@ -1209,7 +1245,15 @@ end class BlockHeadline super Block - redef fun emit(v) do v.decorator.add_headline(v, self) + redef fun emit(v) do + var loc = block.location.copy + loc.column_start += start + v.push_loc(loc) + v.decorator.add_headline(v, self) + v.pop_loc + end + + private var start = 0 # Depth of the headline used to determine the headline level. var depth = 0 @@ -1238,6 +1282,7 @@ class BlockHeadline line.leading = 0 line.trailing = 0 end + self.start = start depth = level.min(6) end end @@ -2495,24 +2540,6 @@ redef class Text return null end - # Init a `MDLocation` instance at `pos` in `self`. - private fun pos_to_loc(pos: Int): MDLocation do - assert pos <= length - var line = 1 - var col = 0 - var i = 0 - while i <= pos do - col += 1 - var c = self[i] - if c == '\n' then - line +=1 - col = 0 - end - i +=1 - end - return new MDLocation(line, col, line, col) - end - # Is `self` an unsafe HTML element? private fun is_html_unsafe: Bool do return html_unsafe_tags.has(self.write_to_string) diff --git a/lib/markdown/test_markdown.nit b/lib/markdown/test_markdown.nit index 6f8519c..5d1b101 100644 --- a/lib/markdown/test_markdown.nit +++ b/lib/markdown/test_markdown.nit @@ -2770,6 +2770,26 @@ class TestTokenLocation "TokenLink at 4,1--4,1"] (new TestTokenProcessor(stack)).process(string) end + + fun test_token_location4 do + var string = "**Hello**\n\n`World`" + var stack = [ + "TokenStrongStar at 1,1--1,1", + "TokenStrongStar at 1,8--1,8", + "TokenCodeSingle at 3,1--3,1", + "TokenCodeSingle at 3,7--3,7"] + (new TestTokenProcessor(stack)).process(string) + end + + fun test_token_location5 do + var string = "# *Title1*\n\n# *Title2*" + var stack = [ + "TokenEmStar at 1,3--1,3", + "TokenEmStar at 1,10--1,10", + "TokenEmStar at 3,3--3,3", + "TokenEmStar at 3,10--3,10"] + (new TestTokenProcessor(stack)).process(string) + end end class TestTokenProcessor @@ -2781,8 +2801,10 @@ class TestTokenProcessor var token = super if token isa TokenNone then return token var res = "{token.class_name} at {token.location}" - print res var exp = test_stack.shift + print "" + print "EXP {exp}" + print "RES {res}" assert exp == res return token end @@ -2822,6 +2844,15 @@ some code proc.emitter.decorator = new TestBlockDecorator(stack) proc.process(string) end + + fun test_block_location3 do + var stack = [ + "BlockHeadline: 1,1--1,8", + "BlockHeadline: 3,1--3,10"] + var string ="""# Title\n\n## Title 2""" + proc.emitter.decorator = new TestBlockDecorator(stack) + proc.process(string) + end end class TestBlockDecorator