X-Git-Url: http://nitlanguage.org

diff --git a/lib/markdown/markdown.nit b/lib/markdown/markdown.nit
index 8ac0b6e..2426d53 100644
--- a/lib/markdown/markdown.nit
+++ b/lib/markdown/markdown.nit
@@ -30,9 +30,6 @@ import template
 # SEE: `String::md_to_html` for a shortcut.
 class MarkdownProcessor
 
-	# `MarkdownEmitter` used for ouput.
-	var emitter: MarkdownEmitter is noinit, protected writable
-
 	# Work in extended mode (default).
 	#
 	# Behavior changes when using extended mode:
@@ -41,33 +38,39 @@ class MarkdownProcessor
 	#
 	#   In normal markdown the following:
 	#
-	#		This is a paragraph
-	#		* and this is not a list
+	# ~~~md
+	# This is a paragraph
+	# * and this is not a list
+	# ~~~
 	#
 	#   Will produce:
 	#
-	#		<p>This is a paragraph
-	#		* and this is not a list</p>
+	# ~~~html
+	# <p>This is a paragraph
+	# * and this is not a list</p>
+	# ~~~
 	#
-	#	When using extended mode this changes to:
+	#   When using extended mode this changes to:
 	#
-	#		<p>This is a paragraph</p>
-	#		<ul>
-	#		<li>and this is not a list</li>
-	#		</ul>
+	# ~~~html
+	# <p>This is a paragraph</p>
+	# <ul>
+	# <li>and this is not a list</li>
+	# </ul>
+	# ~~~
 	#
 	# * Fences code blocks
 	#
 	#   If you don't want to indent your all your code with 4 spaces,
 	#   you can wrap your code in ``` ``` ``` or `~~~`.
 	#
-	#	Here's an example:
+	#   Here's an example:
 	#
-	#		```
-	#		fun test do
-	#			print "Hello World!"
-	#		end
-	#		```
+	# ~~~md
+	# fun test do
+	#    print "Hello World!"
+	# end
+	# ~~~
 	#
 	# * Code blocks meta
 	#
@@ -76,46 +79,64 @@ class MarkdownProcessor
 	#   You can add an optional language identifier after the fence declaration to output
 	#   it in the HTML render.
 	#
-	#		```nit
-	#		import markdown
+	# ```nit
+	# import markdown
 	#
-	#		print "# Hello World!".md_to_html
-	#		```
+	# print "# Hello World!".md_to_html
+	# ```
 	#
 	#   Becomes
 	#
-	#		<pre class="nit"><code>import markdown
+	# ~~~html
+	# <pre class="nit"><code>import markdown
 	#
-	#		print "Hello World!".md_to_html
-	#		</code></pre>
+	# print "Hello World!".md_to_html
+	# </code></pre>
+	# ~~~
 	#
 	# * Underscores (Emphasis)
 	#
 	#   Underscores in the middle of a word like:
 	#
-	#		Con_cat_this
+	# ~~~md
+	# Con_cat_this
+	# ~~~
 	#
-	#	normally produces this:
+	#   normally produces this:
 	#
-	#		<p>Con<em>cat</em>this</p>
+	# ~~~html
+	# <p>Con<em>cat</em>this</p>
+	# ~~~
 	#
 	#   With extended mode they don't result in emphasis.
 	#
-	#		<p>Con_cat_this</p>
+	# ~~~html
+	# <p>Con_cat_this</p>
+	# ~~~
 	#
 	# * Strikethrough
 	#
 	#   Like in [GFM](https://help.github.com/articles/github-flavored-markdown),
 	#   strikethrought span is marked with `~~`.
 	#
-	#		~~Mistaken text.~~
+	# ~~~md
+	# ~~Mistaken text.~~
+	# ~~~
 	#
 	#   becomes
 	#
-	#		<del>Mistaken text.</del>
+	# ~~~html
+	# <del>Mistaken text.</del>
+	# ~~~
 	var ext_mode = true
 
-	init do self.emitter = new MarkdownEmitter(self)
+	# Disable attaching MDLocation to Tokens
+	#
+	# Locations are useful for some tools but they may
+	# cause an important time and space overhead.
+	#
+	# Default = `false`
+	var no_location = false is writable
 
 	# Process the mardown `input` string and return the processed output.
 	fun process(input: String): Writable do
@@ -129,7 +150,8 @@ class MarkdownProcessor
 		parent.remove_surrounding_empty_lines
 		recurse(parent, false)
 		# output processed text
-		return emitter.emit(parent.kind)
+		decorator.headlines.clear
+		return emit(parent.kind)
 	end
 
 	# Split `input` string into `MDLines` and create a parent `MDBlock` with it.
@@ -150,8 +172,9 @@ class MarkdownProcessor
 				var c = input[i]
 				if c == '\n' then
 					eol = true
+				else if c == '\r' then
 				else if c == '\t' then
-					var np = pos + (4 - (pos.bin_and(3)))
+					var np = pos + (4 - (pos & 3))
 					while pos < np do
 						value.add ' '
 						pos += 1
@@ -186,15 +209,15 @@ class MarkdownProcessor
 		if not line.is_empty and line.leading < 4 and line.value[line.leading] == '[' then
 			pos = line.leading + 1
 			pos = md.read_until(id, pos, ']')
-			if not id.is_empty and pos + 2 < line.value.length then
+			if not id.is_empty and pos >= 0 and pos + 2 < line.value.length then
 				if line.value[pos + 1] == ':' then
 					pos += 2
 					pos = md.skip_spaces(pos)
-					if line.value[pos] == '<' then
+					if pos >= 0 and line.value[pos] == '<' then
 						pos += 1
 						pos = md.read_until(link, pos, '>')
 						pos += 1
-					else
+					else if pos >= 0 then
 						pos = md.read_until(link, pos, ' ', '\n')
 					end
 					if not link.is_empty then
@@ -235,7 +258,10 @@ class MarkdownProcessor
 						pos = md.read_until(comment, pos, c)
 					end
 				end
-				if not comment.is_empty then last_link_ref.title = comment.write_to_string
+				var last_link_ref = self.last_link_ref
+				if not comment.is_empty and last_link_ref != null then
+					last_link_ref.title = comment.write_to_string
+				end
 			end
 			if comment.is_empty then return false
 			return true
@@ -250,8 +276,10 @@ class MarkdownProcessor
 	#
 	# Markdown allows link refs to be defined over two lines:
 	#
-	#	[id]: http://example.com/longish/path/to/resource/here
-	#		"Optional Title Here"
+	# ~~~md
+	# [id]: http://example.com/longish/path/to/resource/here
+	#	"Optional Title Here"
+	# ~~~
 	#
 	private var last_link_ref: nullable LinkRef = null
 
@@ -376,7 +404,16 @@ class MarkdownProcessor
 			c2 = ' '
 		end
 
-		var loc = text.pos_to_loc(pos)
+		var loc
+		if no_location then
+			loc = null
+		else
+			loc = new MDLocation(
+				current_loc.line_start,
+				current_loc.column_start + pos,
+				current_loc.line_start,
+				current_loc.column_start + pos)
+		end
 
 		if c == '*' then
 			if c1 == '*' then
@@ -393,7 +430,7 @@ class MarkdownProcessor
 			end
 		else if c == '_' then
 			if c1 == '_' then
-				if c0 != ' ' or c2 != ' 'then
+				if c0 != ' ' or c2 != ' ' then
 					return new TokenStrongUnderscore(loc, pos, c)
 				else
 					return new TokenEmUnderscore(loc, pos, c)
@@ -456,19 +493,6 @@ class MarkdownProcessor
 		end
 		return -1
 	end
-end
-
-# Emit output corresponding to blocks content.
-#
-# Blocks are created by a previous pass in `MarkdownProcessor`.
-# The emitter use a `Decorator` to select the output format.
-class MarkdownEmitter
-
-	# Kind of processor used for parsing.
-	type PROCESSOR: MarkdownProcessor
-
-	# Processor containing link refs.
-	var processor: PROCESSOR
 
 	# Kind of decorator used for decoration.
 	type DECORATOR: Decorator
@@ -480,8 +504,7 @@ class MarkdownEmitter
 	end
 
 	# Create a new `MarkdownEmitter` using a custom `decorator`.
-	init with_decorator(processor: PROCESSOR, decorator: DECORATOR) do
-		init processor
+	init with_decorator(decorator: DECORATOR) do
 		self.decorator = decorator
 	end
 
@@ -499,16 +522,20 @@ class MarkdownEmitter
 	# Transform and emit mardown text
 	fun emit_text(text: Text) do emit_text_until(text, 0, null)
 
-	# Transform and emit mardown text starting at `from` and
+	# Transform and emit mardown text starting at `start` and
 	# until a token with the same type as `token` is found.
-	# Go until the end of text if `token` is null.
+	# Go until the end of `text` if `token` is null.
 	fun emit_text_until(text: Text, start: Int, token: nullable Token): Int do
 		var old_text = current_text
 		var old_pos = current_pos
 		current_text = text
 		current_pos = start
 		while current_pos < text.length do
-			var mt = processor.token_at(text, current_pos)
+			if text[current_pos] == '\n' then
+				current_loc.line_start += 1
+				current_loc.column_start = -current_pos
+			end
+			var mt = token_at(text, current_pos)
 			if (token != null and not token isa TokenNone) and
 			(mt.is_same_type(token) or
 			(token isa TokenEmStar and mt isa TokenStrongStar) or
@@ -550,6 +577,21 @@ class MarkdownEmitter
 		return buffer_stack.last
 	end
 
+	# Stacked locations.
+	private var loc_stack = new List[MDLocation]
+
+	# Push a new MDLocation on the stack.
+	private fun push_loc(location: MDLocation) do loc_stack.add location
+
+	# Pop the last buffer.
+	private fun pop_loc: MDLocation do return loc_stack.pop
+
+	# Current output buffer.
+	private fun current_loc: MDLocation do
+		assert not loc_stack.is_empty
+		return loc_stack.last
+	end
+
 	# Append `e` to current buffer.
 	fun add(e: Writable) do
 		if e isa Text then
@@ -560,10 +602,12 @@ class MarkdownEmitter
 	end
 
 	# Append `c` to current buffer.
-	fun addc(c: Char) do add c.to_s
+	fun addc(c: Char) do
+		current_buffer.add c
+	end
 
 	# Append a "\n" line break.
-	fun addn do add "\n"
+	fun addn do addc '\n'
 end
 
 # A Link Reference.
@@ -585,7 +629,7 @@ class LinkRef
 
 	# Create a link with a title.
 	init with_title(link: String, title: nullable String) do
-		self.link = link
+		init(link)
 		self.title = title
 	end
 end
@@ -594,67 +638,72 @@ end
 # Default decorator used is `HTMLDecorator`.
 interface Decorator
 
-	# Kind of emitter used for decoration.
-	type EMITTER: MarkdownEmitter
+	# Kind of processor used
+	type PROCESSOR: MarkdownProcessor
+
+	# Render a single plain char.
+	#
+	# Redefine this method to add special escaping for plain text.
+	fun add_char(v: PROCESSOR, c: Char) do v.addc c
 
 	# Render a ruler block.
-	fun add_ruler(v: EMITTER, block: BlockRuler) is abstract
+	fun add_ruler(v: PROCESSOR, block: BlockRuler) is abstract
 
 	# Render a headline block with corresponding level.
-	fun add_headline(v: EMITTER, block: BlockHeadline) is abstract
+	fun add_headline(v: PROCESSOR, block: BlockHeadline) is abstract
 
 	# Render a paragraph block.
-	fun add_paragraph(v: EMITTER, block: BlockParagraph) is abstract
+	fun add_paragraph(v: PROCESSOR, block: BlockParagraph) is abstract
 
 	# Render a code or fence block.
-	fun add_code(v: EMITTER, block: BlockCode) is abstract
+	fun add_code(v: PROCESSOR, block: BlockCode) is abstract
 
 	# Render a blockquote.
-	fun add_blockquote(v: EMITTER, block: BlockQuote) is abstract
+	fun add_blockquote(v: PROCESSOR, block: BlockQuote) is abstract
 
 	# Render an unordered list.
-	fun add_unorderedlist(v: EMITTER, block: BlockUnorderedList) is abstract
+	fun add_unorderedlist(v: PROCESSOR, block: BlockUnorderedList) is abstract
 
 	# Render an ordered list.
-	fun add_orderedlist(v: EMITTER, block: BlockOrderedList) is abstract
+	fun add_orderedlist(v: PROCESSOR, block: BlockOrderedList) is abstract
 
 	# Render a list item.
-	fun add_listitem(v: EMITTER, block: BlockListItem) is abstract
+	fun add_listitem(v: PROCESSOR, block: BlockListItem) is abstract
 
 	# Render an emphasis text.
-	fun add_em(v: EMITTER, text: Text) is abstract
+	fun add_em(v: PROCESSOR, text: Text) is abstract
 
 	# Render a strong text.
-	fun add_strong(v: EMITTER, text: Text) is abstract
+	fun add_strong(v: PROCESSOR, text: Text) is abstract
 
 	# Render a strike text.
 	#
 	# Extended mode only (see `MarkdownProcessor::ext_mode`)
-	fun add_strike(v: EMITTER, text: Text) is abstract
+	fun add_strike(v: PROCESSOR, text: Text) is abstract
 
 	# Render a link.
-	fun add_link(v: EMITTER, link: Text, name: Text, comment: nullable Text) is abstract
+	fun add_link(v: PROCESSOR, link: Text, name: Text, comment: nullable Text) is abstract
 
 	# Render an image.
-	fun add_image(v: EMITTER, link: Text, name: Text, comment: nullable Text) is abstract
+	fun add_image(v: PROCESSOR, link: Text, name: Text, comment: nullable Text) is abstract
 
 	# Render an abbreviation.
-	fun add_abbr(v: EMITTER, name: Text, comment: Text) is abstract
+	fun add_abbr(v: PROCESSOR, name: Text, comment: Text) is abstract
 
 	# Render a code span reading from a buffer.
-	fun add_span_code(v: EMITTER, buffer: Text, from, to: Int) is abstract
+	fun add_span_code(v: PROCESSOR, buffer: Text, from, to: Int) is abstract
 
 	# Render a text and escape it.
-	fun append_value(v: EMITTER, value: Text) is abstract
+	fun append_value(v: PROCESSOR, value: Text) is abstract
 
 	# Render code text from buffer and escape it.
-	fun append_code(v: EMITTER, buffer: Text, from, to: Int) is abstract
+	fun append_code(v: PROCESSOR, buffer: Text, from, to: Int) is abstract
 
 	# Render a character escape.
-	fun escape_char(v: EMITTER, char: Char) is abstract
+	fun escape_char(v: PROCESSOR, char: Char) is abstract
 
 	# Render a line break
-	fun add_line_break(v: EMITTER) is abstract
+	fun add_line_break(v: PROCESSOR) is abstract
 
 	# Generate a new html valid id from a `String`.
 	fun strip_id(txt: String): String is abstract
@@ -687,7 +736,9 @@ class HTMLDecorator
 
 	redef fun add_headline(v, block) do
 		# save headline
-		var txt = block.block.first_line.value
+		var line = block.block.first_line
+		if line == null then return
+		var txt = line.value
 		var id = strip_id(txt)
 		var lvl = block.depth
 		headlines[id] = new HeadLine(id, txt, lvl)
@@ -704,8 +755,11 @@ class HTMLDecorator
 	end
 
 	redef fun add_code(v, block) do
-		if block isa BlockFence and block.meta != null then
-			v.add "<pre class=\"{block.meta.to_s}\"><code>"
+		var meta = block.meta
+		if meta != null then
+			v.add "<pre class=\""
+			append_value(v, meta)
+			v.add "\"><code>"
 		else
 			v.add "<pre><code>"
 		end
@@ -880,6 +934,11 @@ class MDLocation
 	var column_end: Int
 
 	redef fun to_s do return "{line_start},{column_start}--{line_end},{column_end}"
+
+	# Return a copy of `self`.
+	fun copy: MDLocation do
+		return new MDLocation(line_start, column_start, line_end, column_end)
+	end
 end
 
 # A block of markdown lines.
@@ -943,8 +1002,8 @@ class MDBlock
 	fun split(line: MDLine): MDBlock do
 		# location for new block
 		var new_loc = new MDLocation(
-			first_line.location.line_start,
-			first_line.location.column_start,
+			first_line.as(not null).location.line_start,
+			first_line.as(not null).location.column_start,
 			line.location.line_end,
 			line.location.column_end)
 		# create block
@@ -956,16 +1015,16 @@ class MDBlock
 		if first_line == null then
 			last_line = null
 		else
-			first_line.prev = null
+			first_line.as(not null).prev = null
 			# update current block loc
-			location.line_start = first_line.location.line_start
-			location.column_start = first_line.location.column_start
+			location.line_start = first_line.as(not null).location.line_start
+			location.column_start = first_line.as(not null).location.column_start
 		end
 		if first_block == null then
 			first_block = block
 			last_block = block
 		else
-			last_block.next = block
+			last_block.as(not null).next = block
 			last_block = block
 		end
 		return block
@@ -977,10 +1036,10 @@ class MDBlock
 			first_line = line
 			last_line = line
 		else
-			last_line.next_empty = line.is_empty
-			line.prev_empty = last_line.is_empty
+			last_line.as(not null).next_empty = line.is_empty
+			line.prev_empty = last_line.as(not null).is_empty
 			line.prev = last_line
-			last_line.next = line
+			last_line.as(not null).next = line
 			last_line = line
 		end
 	end
@@ -990,12 +1049,12 @@ class MDBlock
 		if line.prev == null then
 			first_line = line.next
 		else
-			line.prev.next = line.next
+			line.prev.as(not null).next = line.next
 		end
 		if line.next == null then
 			last_line = line.prev
 		else
-			line.next.prev = line.prev
+			line.next.as(not null).prev = line.prev
 		end
 		line.prev = null
 		line.next = null
@@ -1062,6 +1121,12 @@ class MDBlock
 			text.append "\n"
 			line = line.next
 		end
+		var block = first_block
+		while block != null do
+			text.append block.text
+			text.append "\n"
+			block = block.next
+		end
 		return text.write_to_string
 	end
 end
@@ -1074,10 +1139,10 @@ abstract class Block
 	var block: MDBlock
 
 	# Output `self` using `v.decorator`.
-	fun emit(v: MarkdownEmitter) do v.emit_in(self)
+	fun emit(v: MarkdownProcessor) do v.emit_in(self)
 
 	# Emit the containts of `self`, lines or blocks.
-	fun emit_in(v: MarkdownEmitter) do
+	fun emit_in(v: MarkdownProcessor) do
 		block.remove_surrounding_empty_lines
 		if block.has_lines then
 			emit_lines(v)
@@ -1087,7 +1152,7 @@ abstract class Block
 	end
 
 	# Emit lines contained in `block`.
-	fun emit_lines(v: MarkdownEmitter) do
+	fun emit_lines(v: MarkdownProcessor) do
 		var tpl = v.push_buffer
 		var line = block.first_line
 		while line != null do
@@ -1105,13 +1170,35 @@ abstract class Block
 	end
 
 	# Emit sub-blocks contained in `block`.
-	fun emit_blocks(v: MarkdownEmitter) do
+	fun emit_blocks(v: MarkdownProcessor) do
 		var block = self.block.first_block
 		while block != null do
+			v.push_loc(block.location)
 			block.kind.emit(v)
+			v.pop_loc
 			block = block.next
 		end
 	end
+
+	# The raw content of the block as a multi-line string.
+	fun raw_content: String do
+		var infence = self isa BlockFence
+		var text = new FlatBuffer
+		var line = self.block.first_line
+		while line != null do
+			if not line.is_empty then
+				var str = line.value
+				if not infence and str.has_prefix("    ") then
+					text.append str.substring(4, str.length - line.trailing)
+				else
+					text.append str
+				end
+			end
+			text.append "\n"
+			line = line.next
+		end
+		return text.write_to_string
+	end
 end
 
 # A block without any markdown specificities.
@@ -1152,6 +1239,9 @@ end
 class BlockCode
 	super Block
 
+	# Any string found after fence token.
+	var meta: nullable Text
+
 	# Number of char to skip at the beginning of the line.
 	#
 	# Block code lines start at 4 spaces.
@@ -1178,9 +1268,6 @@ end
 class BlockFence
 	super BlockCode
 
-	# Any string found after fence token.
-	var meta: nullable Text
-
 	# Fence code lines start at 0 spaces.
 	redef var line_start = 0
 end
@@ -1189,7 +1276,15 @@ end
 class BlockHeadline
 	super Block
 
-	redef fun emit(v) do v.decorator.add_headline(v, self)
+	redef fun emit(v) do
+		var loc = block.location.copy
+		loc.column_start += start
+		v.push_loc(loc)
+		v.decorator.add_headline(v, self)
+		v.pop_loc
+	end
+
+	private var start = 0
 
 	# Depth of the headline used to determine the headline level.
 	var depth = 0
@@ -1199,6 +1294,7 @@ class BlockHeadline
 		if depth > 0 then return
 		var level = 0
 		var line = block.first_line
+		if line == null then return
 		if line.is_empty then return
 		var start = line.leading
 		while start < line.value.length and line.value[start] == '#' do
@@ -1218,6 +1314,7 @@ class BlockHeadline
 			line.leading = 0
 			line.trailing = 0
 		end
+		self.start = start
 		depth = level.min(6)
 	end
 end
@@ -1237,6 +1334,7 @@ abstract class BlockList
 	# Split list block into list items sub-blocks.
 	private fun init_block(v: MarkdownProcessor) do
 		var line = block.first_line
+		if line == null then return
 		line = line.next
 		while line != null do
 			var t = v.line_kind(line)
@@ -1369,8 +1467,8 @@ class MDLine
 		leading = 0
 		trailing = 0
 		is_empty = true
-		if prev != null then prev.next_empty = true
-		if next != null then next.prev_empty = true
+		if prev != null then prev.as(not null).next_empty = true
+		if next != null then next.as(not null).prev_empty = true
 	end
 
 	# Number or leading spaces on this line.
@@ -1504,8 +1602,8 @@ class MDLine
 	# Used by `check_html`.
 	private fun read_xml_comment(first_line: MDLine, start: Int): Int do
 		var line: nullable MDLine = first_line
-		if start + 3 < line.value.length then
-			if line.value[2] == '-' and line.value[3] == '-' then
+		if start + 3 < line.as(not null).value.length then
+			if line.as(not null).value[2] == '-' and line.as(not null).value[3] == '-' then
 				var pos = start + 4
 				while line != null do
 					while pos < line.value.length and line.value[pos] != '-' do
@@ -1546,7 +1644,7 @@ class LineEmpty
 	super Line
 
 	redef fun process(v) do
-		v.current_line = v.current_line.next
+		v.current_line = v.current_line.as(not null).next
 	end
 end
 
@@ -1558,7 +1656,7 @@ class LineOther
 	redef fun process(v) do
 		var line = v.current_line
 		# go to block end
-		var was_empty = line.prev_empty
+		var was_empty = line.as(not null).prev_empty
 		while line != null and not line.is_empty do
 			var t = v.line_kind(line)
 			if (v.in_list or v.ext_mode) and t isa LineList then
@@ -1574,29 +1672,30 @@ class LineOther
 			line = line.next
 		end
 		# build block
+		var current_block = v.current_block.as(not null)
 		if line != null and not line.is_empty then
-			var block = v.current_block.split(line.prev.as(not null))
+			var block = current_block.split(line.prev.as(not null))
 			if v.in_list and not was_empty then
 				block.kind = new BlockNone(block)
 			else
 				block.kind = new BlockParagraph(block)
 			end
-			v.current_block.remove_leading_empty_lines
+			current_block.remove_leading_empty_lines
 		else
 			var block: MDBlock
 			if line != null then
-				block = v.current_block.split(line)
+				block = current_block.split(line)
 			else
-				block = v.current_block.split(v.current_block.last_line.as(not null))
+				block = current_block.split(current_block.last_line.as(not null))
 			end
 			if v.in_list and (line == null or not line.is_empty) and not was_empty then
 				block.kind = new BlockNone(block)
 			else
 				block.kind = new BlockParagraph(block)
 			end
-			v.current_block.remove_leading_empty_lines
+			current_block.remove_leading_empty_lines
 		end
-		v.current_line = v.current_block.first_line
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1611,15 +1710,16 @@ class LineCode
 			line = line.next
 		end
 		# split at block end line
+		var current_block = v.current_block.as(not null)
 		var block: MDBlock
 		if line != null then
-			block = v.current_block.split(line.prev.as(not null))
+			block = current_block.split(line.prev.as(not null))
 		else
-			block = v.current_block.split(v.current_block.last_line.as(not null))
+			block = current_block.split(current_block.last_line.as(not null))
 		end
 		block.kind = new BlockCode(block)
 		block.remove_surrounding_empty_lines
-		v.current_line = v.current_block.first_line
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1629,12 +1729,14 @@ class LineXML
 
 	redef fun process(v) do
 		var line = v.current_line
+		if line == null then return
+		var current_block = v.current_block.as(not null)
 		var prev = line.prev
-		if prev != null then v.current_block.split(prev)
-		var block = v.current_block.split(line.xml_end_line.as(not null))
+		if prev != null then current_block.split(prev)
+		var block = current_block.split(line.xml_end_line.as(not null))
 		block.kind = new BlockXML(block)
-		v.current_block.remove_leading_empty_lines
-		v.current_line = v.current_block.first_line
+		current_block.remove_leading_empty_lines
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1644,6 +1746,7 @@ class LineBlockquote
 
 	redef fun process(v) do
 		var line = v.current_line
+		var current_block = v.current_block.as(not null)
 		# go to bquote end
 		while line != null do
 			if not line.is_empty and (line.prev_empty and
@@ -1654,9 +1757,9 @@ class LineBlockquote
 		# build sub block
 		var block: MDBlock
 		if line != null then
-			block = v.current_block.split(line.prev.as(not null))
+			block = current_block.split(line.prev.as(not null))
 		else
-			block = v.current_block.split(v.current_block.last_line.as(not null))
+			block = current_block.split(current_block.last_line.as(not null))
 		end
 		var kind = new BlockQuote(block)
 		block.kind = kind
@@ -1664,7 +1767,7 @@ class LineBlockquote
 		kind.remove_block_quote_prefix(block)
 		v.current_line = line
 		v.recurse(block, false)
-		v.current_line = v.current_block.first_line
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1674,11 +1777,13 @@ class LineHR
 
 	redef fun process(v) do
 		var line = v.current_line
-		if line.prev != null then v.current_block.split(line.prev.as(not null))
-		var block = v.current_block.split(line.as(not null))
+		if line == null then return
+		var current_block = v.current_block.as(not null)
+		if line.prev != null then current_block.split(line.prev.as(not null))
+		var block = current_block.split(line)
 		block.kind = new BlockRuler(block)
-		v.current_block.remove_leading_empty_lines
-		v.current_line = v.current_block.first_line
+		current_block.remove_leading_empty_lines
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1688,7 +1793,8 @@ class LineFence
 
 	redef fun process(v) do
 		# go to fence end
-		var line = v.current_line.next
+		var line = v.current_line.as(not null).next
+		var current_block = v.current_block.as(not null)
 		while line != null do
 			if v.line_kind(line) isa LineFence then break
 			line = line.next
@@ -1699,17 +1805,17 @@ class LineFence
 		# build fence block
 		var block: MDBlock
 		if line != null then
-			block = v.current_block.split(line.prev.as(not null))
+			block = current_block.split(line.prev.as(not null))
 		else
-			block = v.current_block.split(v.current_block.last_line.as(not null))
+			block = current_block.split(current_block.last_line.as(not null))
 		end
 		block.remove_surrounding_empty_lines
-		var meta = block.first_line.value.meta_from_fence
+		var meta = block.first_line.as(not null).value.meta_from_fence
 		block.kind = new BlockFence(block, meta)
-		block.first_line.clear
+		block.first_line.as(not null).clear
 		var last = block.last_line
 		if last != null and v.line_kind(last) isa LineFence then
-			block.last_line.clear
+			block.last_line.as(not null).clear
 		end
 		block.remove_surrounding_empty_lines
 		v.current_line = line
@@ -1722,14 +1828,16 @@ class LineHeadline
 
 	redef fun process(v) do
 		var line = v.current_line
+		if line == null then return
+		var current_block = v.current_block.as(not null)
 		var lprev = line.prev
-		if lprev != null then v.current_block.split(lprev)
-		var block = v.current_block.split(line.as(not null))
+		if lprev != null then current_block.split(lprev)
+		var block = current_block.split(line)
 		var kind = new BlockHeadline(block)
 		block.kind = kind
 		kind.transform_headline(block)
-		v.current_block.remove_leading_empty_lines
-		v.current_line = v.current_block.first_line
+		current_block.remove_leading_empty_lines
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1739,16 +1847,18 @@ class LineHeadline1
 
 	redef fun process(v) do
 		var line = v.current_line
+		if line == null then return
+		var current_block = v.current_block.as(not null)
 		var lprev = line.prev
-		if lprev != null then v.current_block.split(lprev)
-		line.next.clear
-		var block = v.current_block.split(line.as(not null))
+		if lprev != null then current_block.split(lprev)
+		line.next.as(not null).clear
+		var block = current_block.split(line)
 		var kind = new BlockHeadline(block)
 		kind.depth = 1
 		kind.transform_headline(block)
 		block.kind = kind
-		v.current_block.remove_leading_empty_lines
-		v.current_line = v.current_block.first_line
+		current_block.remove_leading_empty_lines
+		v.current_line = current_block.first_line
 	end
 end
 
@@ -1758,22 +1868,24 @@ class LineHeadline2
 
 	redef fun process(v) do
 		var line = v.current_line
+		if line == null then return
+		var current_block = v.current_block.as(not null)
 		var lprev = line.prev
-		if lprev != null then v.current_block.split(lprev)
-		line.next.clear
-		var block = v.current_block.split(line.as(not null))
+		if lprev != null then current_block.split(lprev)
+		line.next.as(not null).clear
+		var block = current_block.split(line)
 		var kind = new BlockHeadline(block)
 		kind.depth = 2
 		kind.transform_headline(block)
 		block.kind = kind
-		v.current_block.remove_leading_empty_lines
-		v.current_line = v.current_block.first_line
+		current_block.remove_leading_empty_lines
+		v.current_line = current_block.first_line
 	end
 end
 
 # A markdown list line.
 # Mainly used to factorize code between ordered and unordered lists.
-class LineList
+abstract class LineList
 	super Line
 
 	redef fun process(v) do
@@ -1786,19 +1898,20 @@ class LineList
 			line = line.next
 		end
 		# build list block
+		var current_block = v.current_block.as(not null)
 		var list: MDBlock
 		if line != null then
-			list = v.current_block.split(line.prev.as(not null))
+			list = current_block.split(line.prev.as(not null))
 		else
-			list = v.current_block.split(v.current_block.last_line.as(not null))
+			list = current_block.split(current_block.last_line.as(not null))
 		end
 		var kind = block_kind(list)
 		list.kind = kind
-		list.first_line.prev_empty = false
-		list.last_line.next_empty = false
+		list.first_line.as(not null).prev_empty = false
+		list.last_line.as(not null).next_empty = false
 		list.remove_surrounding_empty_lines
-		list.first_line.prev_empty = false
-		list.last_line.next_empty = false
+		list.first_line.as(not null).prev_empty = false
+		list.last_line.as(not null).next_empty = false
 		kind.init_block(v)
 		var block = list.first_block
 		while block != null do
@@ -1844,7 +1957,7 @@ end
 abstract class Token
 
 	# Location of `self` in the original input.
-	var location: MDLocation
+	var location: nullable MDLocation
 
 	# Position of `self` in input independant from lines.
 	var pos: Int
@@ -1853,7 +1966,7 @@ abstract class Token
 	var char: Char
 
 	# Output that token using `MarkdownEmitter::decorator`.
-	fun emit(v: MarkdownEmitter) do v.addc char
+	fun emit(v: MarkdownProcessor) do v.decorator.add_char(v, char)
 end
 
 # A token without a specific meaning.
@@ -1921,14 +2034,15 @@ abstract class TokenCode
 	super Token
 
 	redef fun emit(v) do
+		var current_text = v.current_text.as(not null)
 		var a = pos + next_pos + 1
-		var b = v.processor.find_token(v.current_text.as(not null), a, self)
+		var b = v.find_token(current_text, a, self)
 		if b > 0 then
 			v.current_pos = b + next_pos
-			while a < b and v.current_text[a] == ' ' do a += 1
+			while a < b and current_text[a] == ' ' do a += 1
 			if a < b then
-				while v.current_text[b - 1] == ' ' do b -= 1
-				v.decorator.add_span_code(v, v.current_text.as(not null), a, b)
+				while current_text[b - 1] == ' ' do b -= 1
+				v.decorator.add_span_code(v, current_text, a, b)
 			end
 		else
 			v.addc char
@@ -1981,11 +2095,12 @@ abstract class TokenLinkOrImage
 	end
 
 	# Emit the hyperlink as link or image.
-	private fun emit_hyper(v: MarkdownEmitter) is abstract
+	private fun emit_hyper(v: MarkdownProcessor) is abstract
 
 	# Check if the link is a valid link.
-	private fun check_link(v: MarkdownEmitter, out: FlatBuffer, start: Int, token: Token): Int do
+	private fun check_link(v: MarkdownProcessor, out: FlatBuffer, start: Int, token: Token): Int do
 		var md = v.current_text
+		if md == null then return -1
 		var pos
 		if token isa TokenLink then
 			pos = start + 1
@@ -2000,9 +2115,9 @@ abstract class TokenLinkOrImage
 		pos += 1
 		pos = md.skip_spaces(pos)
 		if pos < start then
-			var tid = name.write_to_string.to_lower
-			if v.processor.link_refs.has_key(tid) then
-				var lr = v.processor.link_refs[tid]
+			var tid = name.as(not null).write_to_string.to_lower
+			if v.link_refs.has_key(tid) then
+				var lr = v.link_refs[tid]
 				is_abbrev = lr.is_abbrev
 				link = lr.link
 				comment = lr.title
@@ -2037,6 +2152,7 @@ abstract class TokenLinkOrImage
 					if pos == -1 then return -1
 				end
 			end
+			if pos < start then return -1
 			if md[pos] != ')' then return -1
 		else if md[pos] == '[' then
 			pos += 1
@@ -2049,16 +2165,16 @@ abstract class TokenLinkOrImage
 			else
 				id = name
 			end
-			var tid = id.write_to_string.to_lower
-			if v.processor.link_refs.has_key(tid) then
-				var lr = v.processor.link_refs[tid]
+			var tid = id.as(not null).write_to_string.to_lower
+			if v.link_refs.has_key(tid) then
+				var lr = v.link_refs[tid]
 				link = lr.link
 				comment = lr.title
 			end
 		else
-			var tid = name.write_to_string.replace("\n", " ").to_lower
-			if v.processor.link_refs.has_key(tid) then
-				var lr = v.processor.link_refs[tid]
+			var tid = name.as(not null).write_to_string.replace("\n", " ").to_lower
+			if v.link_refs.has_key(tid) then
+				var lr = v.link_refs[tid]
 				link = lr.link
 				comment = lr.title
 				pos = old_pos
@@ -2110,7 +2226,7 @@ class TokenHTML
 
 	# Is the HTML valid?
 	# Also take care of link and mailto shortcuts.
-	private fun check_html(v: MarkdownEmitter, out: FlatBuffer, md: Text, start: Int): Int do
+	private fun check_html(v: MarkdownProcessor, out: FlatBuffer, md: Text, start: Int): Int do
 		# check for auto links
 		var tmp = new FlatBuffer
 		var pos = md.read_until(tmp, start + 1, ':', ' ', '>', '\n')
@@ -2191,7 +2307,7 @@ class TokenEscape
 
 	redef fun emit(v) do
 		v.current_pos += 1
-		v.addc v.current_text[v.current_pos]
+		v.addc v.current_text.as(not null)[v.current_pos]
 	end
 end
 
@@ -2235,18 +2351,11 @@ redef class Text
 			if c == '\\' and pos + 1 < length then
 				pos = escape(out, self[pos + 1], pos)
 			else
-				var end_reached = false
-				for n in nend do
-					if c == n then
-						end_reached = true
-						break
-					end
-				end
-				if end_reached then break
+				for n in nend do if c == n then break label
 				out.add c
 			end
 			pos += 1
-		end
+		end label
 		if pos == length then return -1
 		return pos
 	end
@@ -2475,24 +2584,6 @@ redef class Text
 		return null
 	end
 
-	# Init a `MDLocation` instance at `pos` in `self`.
-	private fun pos_to_loc(pos: Int): MDLocation do
-		assert pos <= length
-		var line = 1
-		var col = 0
-		var i = 0
-		while i <= pos do
-			col += 1
-			var c = self[i]
-			if c == '\n' then
-				line +=1
-				col = 0
-			end
-			i +=1
-		end
-		return new MDLocation(line, col, line, col)
-	end
-
 	# Is `self` an unsafe HTML element?
 	private fun is_html_unsafe: Bool do return html_unsafe_tags.has(self.write_to_string)