X-Git-Url: http://nitlanguage.org

diff --git a/lib/markdown/markdown.nit b/lib/markdown/markdown.nit
index 4efde74..613e1a4 100644
--- a/lib/markdown/markdown.nit
+++ b/lib/markdown/markdown.nit
@@ -30,12 +30,113 @@ import template
 # SEE: `String::md_to_html` for a shortcut.
 class MarkdownProcessor
 
-	var emitter: MarkdownEmitter is noinit
+	# `MarkdownEmitter` used for ouput.
+	var emitter: MarkdownEmitter is noinit, protected writable
+
+	# Work in extended mode (default).
+	#
+	# Behavior changes when using extended mode:
+	#
+	# * Lists and code blocks end a paragraph
+	#
+	#   In normal markdown the following:
+	#
+	# ~~~md
+	# This is a paragraph
+	# * and this is not a list
+	# ~~~
+	#
+	#   Will produce:
+	#
+	# ~~~html
+	# <p>This is a paragraph
+	# * and this is not a list</p>
+	# ~~~
+	#
+	#   When using extended mode this changes to:
+	#
+	# ~~~html
+	# <p>This is a paragraph</p>
+	# <ul>
+	# <li>and this is not a list</li>
+	# </ul>
+	# ~~~
+	#
+	# * Fences code blocks
+	#
+	#   If you don't want to indent your all your code with 4 spaces,
+	#   you can wrap your code in ``` ``` ``` or `~~~`.
+	#
+	#   Here's an example:
+	#
+	# ~~~md
+	# fun test do
+	#    print "Hello World!"
+	# end
+	# ~~~
+	#
+	# * Code blocks meta
+	#
+	#   If you want to use syntax highlighting tools, most of them need to know what kind
+	#   of language they are highlighting.
+	#   You can add an optional language identifier after the fence declaration to output
+	#   it in the HTML render.
+	#
+	# ```nit
+	# import markdown
+	#
+	# print "# Hello World!".md_to_html
+	# ```
+	#
+	#   Becomes
+	#
+	# ~~~html
+	# <pre class="nit"><code>import markdown
+	#
+	# print "Hello World!".md_to_html
+	# </code></pre>
+	# ~~~
+	#
+	# * Underscores (Emphasis)
+	#
+	#   Underscores in the middle of a word like:
+	#
+	# ~~~md
+	# Con_cat_this
+	# ~~~
+	#
+	#   normally produces this:
+	#
+	# ~~~html
+	# <p>Con<em>cat</em>this</p>
+	# ~~~
+	#
+	#   With extended mode they don't result in emphasis.
+	#
+	# ~~~html
+	# <p>Con_cat_this</p>
+	# ~~~
+	#
+	# * Strikethrough
+	#
+	#   Like in [GFM](https://help.github.com/articles/github-flavored-markdown),
+	#   strikethrought span is marked with `~~`.
+	#
+	# ~~~md
+	# ~~Mistaken text.~~
+	# ~~~
+	#
+	#   becomes
+	#
+	# ~~~html
+	# <del>Mistaken text.</del>
+	# ~~~
+	var ext_mode = true
 
 	init do self.emitter = new MarkdownEmitter(self)
 
 	# Process the mardown `input` string and return the processed output.
-	fun process(input: String): Streamable do
+	fun process(input: String): Writable do
 		# init processor
 		link_refs.clear
 		last_link_ref = null
@@ -51,36 +152,42 @@ class MarkdownProcessor
 
 	# Split `input` string into `MDLines` and create a parent `MDBlock` with it.
 	private fun read_lines(input: String): MDBlock do
-		var block = new MDBlock
+		var block = new MDBlock(new MDLocation(1, 1, 1, 1))
 		var value = new FlatBuffer
 		var i = 0
+
+		var line_pos = 0
+		var col_pos = 0
+
 		while i < input.length do
 			value.clear
 			var pos = 0
 			var eol = false
 			while not eol and i < input.length do
+				col_pos += 1
 				var c = input[i]
 				if c == '\n' then
-					i += 1
 					eol = true
 				else if c == '\t' then
-					var np = pos + (4 - (pos.bin_and(3)))
+					var np = pos + (4 - (pos & 3))
 					while pos < np do
 						value.add ' '
 						pos += 1
 					end
-					i += 1
 				else
 					pos += 1
 					value.add c
-					i += 1
 				end
+				i += 1
 			end
+			line_pos += 1
 
-			var line = new MDLine(value.write_to_string)
+			var loc = new MDLocation(line_pos, 1, line_pos, col_pos)
+			var line = new MDLine(loc, value.write_to_string)
 			var is_link_ref = check_link_ref(line)
 			# Skip link refs
 			if not is_link_ref then block.add_line line
+			col_pos = 0
 		end
 		return block
 	end
@@ -161,8 +268,10 @@ class MarkdownProcessor
 	#
 	# Markdown allows link refs to be defined over two lines:
 	#
-	# 	[id]: http://example.com/longish/path/to/resource/here
-	#		"Optional Title Here"
+	# ~~~md
+	# [id]: http://example.com/longish/path/to/resource/here
+	#	"Optional Title Here"
+	# ~~~
 	#
 	private var last_link_ref: nullable LinkRef = null
 
@@ -189,7 +298,7 @@ class MarkdownProcessor
 		current_line = line
 		current_block = root
 		while current_line != null do
-			current_line.kind(self).process(self)
+			line_kind(current_line.as(not null)).process(self)
 		end
 		self.in_list = old_mode
 		self.current_block = old_root
@@ -206,6 +315,177 @@ class MarkdownProcessor
 	# Is the current recursion in list mode?
 	# Used when visiting blocks with `recurse`
 	private var in_list = false
+
+	# The type of line.
+	# see: `md_line_*`
+	fun line_kind(md: MDLine): Line do
+		var value = md.value
+		var leading = md.leading
+		var trailing = md.trailing
+		if md.is_empty then return new LineEmpty
+		if md.leading > 3 then return new LineCode
+		if value[leading] == '#' then return new LineHeadline
+		if value[leading] == '>' then return new LineBlockquote
+
+		if ext_mode then
+			if value.length - leading - trailing > 2 then
+				if value[leading] == '`' and md.count_chars_start('`') >= 3 then
+					return new LineFence
+				end
+				if value[leading] == '~' and md.count_chars_start('~') >= 3 then
+					return new LineFence
+				end
+			end
+		end
+
+		if value.length - leading - trailing > 2 and
+		   (value[leading] == '*' or value[leading] == '-' or value[leading] == '_') then
+		   if md.count_chars(value[leading]) >= 3 then
+				return new LineHR
+		   end
+		end
+
+		if value.length - leading >= 2 and value[leading + 1] == ' ' then
+			var c = value[leading]
+			if c == '*' or c == '-' or c == '+' then return new LineUList
+		end
+
+		if value.length - leading >= 3 and value[leading].is_digit then
+			var i = leading + 1
+			while i < value.length and value[i].is_digit do i += 1
+			if i + 1 < value.length and value[i] == '.' and value[i + 1] == ' ' then
+				return new LineOList
+			end
+		end
+
+		if value[leading] == '<' and md.check_html then return new LineXML
+
+		var next = md.next
+		if next != null and not next.is_empty then
+			if next.count_chars('=') > 0 then
+				return new LineHeadline1
+			end
+			if next.count_chars('-') > 0 then
+				return new LineHeadline2
+			end
+		end
+		return new LineOther
+	end
+
+	# Get the token kind at `pos`.
+	fun token_at(text: Text, pos: Int): Token do
+		var c0: Char
+		var c1: Char
+		var c2: Char
+
+		if pos > 0 then
+			c0 = text[pos - 1]
+		else
+			c0 = ' '
+		end
+		var c = text[pos]
+
+		if pos + 1 < text.length then
+			c1 = text[pos + 1]
+		else
+			c1 = ' '
+		end
+		if pos + 2 < text.length then
+			c2 = text[pos + 2]
+		else
+			c2 = ' '
+		end
+
+		var loc = new MDLocation(
+			current_loc.line_start,
+			current_loc.column_start + pos,
+			current_loc.line_start,
+			current_loc.column_start + pos)
+
+		if c == '*' then
+			if c1 == '*' then
+				if c0 != ' ' or c2 != ' ' then
+					return new TokenStrongStar(loc, pos, c)
+				else
+					return new TokenEmStar(loc, pos, c)
+				end
+			end
+			if c0 != ' ' or c1 != ' ' then
+				return new TokenEmStar(loc, pos, c)
+			else
+				return new TokenNone(loc, pos, c)
+			end
+		else if c == '_' then
+			if c1 == '_' then
+				if c0 != ' ' or c2 != ' 'then
+					return new TokenStrongUnderscore(loc, pos, c)
+				else
+					return new TokenEmUnderscore(loc, pos, c)
+				end
+			end
+			if ext_mode then
+				if (c0.is_letter or c0.is_digit) and c0 != '_' and
+				   (c1.is_letter or c1.is_digit) then
+					return new TokenNone(loc, pos, c)
+				else
+					return new TokenEmUnderscore(loc, pos, c)
+				end
+			end
+			if c0 != ' ' or c1 != ' ' then
+				return new TokenEmUnderscore(loc, pos, c)
+			else
+				return new TokenNone(loc, pos, c)
+			end
+		else if c == '!' then
+			if c1 == '[' then return new TokenImage(loc, pos, c)
+			return new TokenNone(loc, pos, c)
+		else if c == '[' then
+			return new TokenLink(loc, pos, c)
+		else if c == ']' then
+			return new TokenNone(loc, pos, c)
+		else if c == '`' then
+			if c1 == '`' then
+				return new TokenCodeDouble(loc, pos, c)
+			else
+				return new TokenCodeSingle(loc, pos, c)
+			end
+		else if c == '\\' then
+			if c1 == '\\' or c1 == '[' or c1 == ']' or c1 == '(' or c1 == ')' or c1 == '{' or c1 == '}' or c1 == '#' or c1 == '"' or c1 == '\'' or c1 == '.' or c1 == '<' or c1 == '>' or c1 == '*' or c1 == '+' or c1 == '-' or c1 == '_' or c1 == '!' or c1 == '`' or c1 == '~' or c1 == '^' then
+				return new TokenEscape(loc, pos, c)
+			else
+				return new TokenNone(loc, pos, c)
+			end
+		else if c == '<' then
+			return new TokenHTML(loc, pos, c)
+		else if c == '&' then
+			return new TokenEntity(loc, pos, c)
+		else
+			if ext_mode then
+				if c == '~' and c1 == '~' then
+					return new TokenStrike(loc, pos, c)
+				end
+			end
+			return new TokenNone(loc, pos, c)
+		end
+	end
+
+	# Find the position of a `token` in `self`.
+	fun find_token(text: Text, start: Int, token: Token): Int do
+		var pos = start
+		while pos < text.length do
+			if token_at(text, pos).is_same_type(token) then
+				return pos
+			end
+			pos += 1
+		end
+		return -1
+	end
+
+	# Location used for next parsed token.
+	#
+	# This location can be changed by the emitter to adjust with `\n` found
+	# in the input.
+	private fun current_loc: MDLocation do return emitter.current_loc
 end
 
 # Emit output corresponding to blocks content.
@@ -214,20 +494,23 @@ end
 # The emitter use a `Decorator` to select the output format.
 class MarkdownEmitter
 
+	# Kind of processor used for parsing.
+	type PROCESSOR: MarkdownProcessor
+
 	# Processor containing link refs.
-	var processor: MarkdownProcessor
+	var processor: PROCESSOR
+
+	# Kind of decorator used for decoration.
+	type DECORATOR: Decorator
 
 	# Decorator used for output.
 	# Default is `HTMLDecorator`
-	var decorator: Decorator = new HTMLDecorator is writable
-
-	# Create a new `MardownEmitter` using the default `HTMLDecorator`
-	init(processor: MarkdownProcessor) do
-		self.processor = processor
+	var decorator: DECORATOR is writable, lazy do
+		return new HTMLDecorator
 	end
 
 	# Create a new `MarkdownEmitter` using a custom `decorator`.
-	init with_decorator(processor: MarkdownProcessor, decorator: Decorator) do
+	init with_decorator(processor: PROCESSOR, decorator: DECORATOR) do
 		init processor
 		self.decorator = decorator
 	end
@@ -244,20 +527,22 @@ class MarkdownEmitter
 	fun emit_in(block: Block) do block.emit_in(self)
 
 	# Transform and emit mardown text
-	fun emit_text(text: Text) do
-		emit_text_until(text, 0, null)
-	end
+	fun emit_text(text: Text) do emit_text_until(text, 0, null)
 
-	# Transform and emit mardown text starting at `from` and
+	# Transform and emit mardown text starting at `start` and
 	# until a token with the same type as `token` is found.
-	# Go until the end of text if `token` is null.
+	# Go until the end of `text` if `token` is null.
 	fun emit_text_until(text: Text, start: Int, token: nullable Token): Int do
 		var old_text = current_text
 		var old_pos = current_pos
 		current_text = text
 		current_pos = start
 		while current_pos < text.length do
-			var mt = text.token_at(current_pos)
+			if text[current_pos] == '\n' then
+				current_loc.line_start += 1
+				current_loc.column_start = -current_pos
+			end
+			var mt = processor.token_at(text, current_pos)
 			if (token != null and not token isa TokenNone) and
 			(mt.is_same_type(token) or
 			(token isa TokenEmStar and mt isa TokenStrongStar) or
@@ -299,8 +584,23 @@ class MarkdownEmitter
 		return buffer_stack.last
 	end
 
+	# Stacked locations.
+	private var loc_stack = new List[MDLocation]
+
+	# Push a new MDLocation on the stack.
+	private fun push_loc(location: MDLocation) do loc_stack.add location
+
+	# Pop the last buffer.
+	private fun pop_loc: MDLocation do return loc_stack.pop
+
+	# Current output buffer.
+	private fun current_loc: MDLocation do
+		assert not loc_stack.is_empty
+		return loc_stack.last
+	end
+
 	# Append `e` to current buffer.
-	fun add(e: Streamable) do
+	fun add(e: Writable) do
 		if e isa Text then
 			current_buffer.append e
 		else
@@ -309,18 +609,18 @@ class MarkdownEmitter
 	end
 
 	# Append `c` to current buffer.
-	fun addc(c: Char) do current_buffer.add c
+	fun addc(c: Char) do add c.to_s
 
 	# Append a "\n" line break.
-	fun addn do current_buffer.add '\n'
+	fun addn do add "\n"
 end
 
 # A Link Reference.
 # Links that are specified somewhere in the mardown document to be reused as shortcuts.
 #
-# Example:
-#
-#    [1]: http://example.com/ "Optional title"
+# ~~~raw
+# [1]: http://example.com/ "Optional title"
+# ~~~
 class LinkRef
 
 	# Link href
@@ -332,6 +632,7 @@ class LinkRef
 	# Is the link an abreviation?
 	var is_abbrev = false
 
+	# Create a link with a title.
 	init with_title(link: String, title: nullable String) do
 		self.link = link
 		self.title = title
@@ -342,62 +643,72 @@ end
 # Default decorator used is `HTMLDecorator`.
 interface Decorator
 
+	# Kind of emitter used for decoration.
+	type EMITTER: MarkdownEmitter
+
+	# Render a single plain char.
+	#
+	# Redefine this method to add special escaping for plain text.
+	fun add_char(v: EMITTER, c: Char) do v.addc c
+
 	# Render a ruler block.
-	fun add_ruler(v: MarkdownEmitter, block: BlockRuler) is abstract
+	fun add_ruler(v: EMITTER, block: BlockRuler) is abstract
 
 	# Render a headline block with corresponding level.
-	fun add_headline(v: MarkdownEmitter, block: BlockHeadline) is abstract
+	fun add_headline(v: EMITTER, block: BlockHeadline) is abstract
 
 	# Render a paragraph block.
-	fun add_paragraph(v: MarkdownEmitter, block: BlockParagraph) is abstract
+	fun add_paragraph(v: EMITTER, block: BlockParagraph) is abstract
 
 	# Render a code or fence block.
-	fun add_code(v: MarkdownEmitter, block: BlockCode) is abstract
+	fun add_code(v: EMITTER, block: BlockCode) is abstract
 
 	# Render a blockquote.
-	fun add_blockquote(v: MarkdownEmitter, block: BlockQuote) is abstract
+	fun add_blockquote(v: EMITTER, block: BlockQuote) is abstract
 
 	# Render an unordered list.
-	fun add_unorderedlist(v: MarkdownEmitter, block: BlockUnorderedList) is abstract
+	fun add_unorderedlist(v: EMITTER, block: BlockUnorderedList) is abstract
 
 	# Render an ordered list.
-	fun add_orderedlist(v: MarkdownEmitter, block: BlockOrderedList) is abstract
+	fun add_orderedlist(v: EMITTER, block: BlockOrderedList) is abstract
 
 	# Render a list item.
-	fun add_listitem(v: MarkdownEmitter, block: BlockListItem) is abstract
+	fun add_listitem(v: EMITTER, block: BlockListItem) is abstract
 
 	# Render an emphasis text.
-	fun add_em(v: MarkdownEmitter, text: Text) is abstract
+	fun add_em(v: EMITTER, text: Text) is abstract
 
 	# Render a strong text.
-	fun add_strong(v: MarkdownEmitter, text: Text) is abstract
+	fun add_strong(v: EMITTER, text: Text) is abstract
 
-	# Render a super text.
-	fun add_super(v: MarkdownEmitter, text: Text) is abstract
+	# Render a strike text.
+	#
+	# Extended mode only (see `MarkdownProcessor::ext_mode`)
+	fun add_strike(v: EMITTER, text: Text) is abstract
 
 	# Render a link.
-	fun add_link(v: MarkdownEmitter, link: Text, name: Text, comment: nullable Text) is abstract
+	fun add_link(v: EMITTER, link: Text, name: Text, comment: nullable Text) is abstract
 
 	# Render an image.
-	fun add_image(v: MarkdownEmitter, link: Text, name: Text, comment: nullable Text) is abstract
+	fun add_image(v: EMITTER, link: Text, name: Text, comment: nullable Text) is abstract
 
 	# Render an abbreviation.
-	fun add_abbr(v: MarkdownEmitter, name: Text, comment: Text) is abstract
+	fun add_abbr(v: EMITTER, name: Text, comment: Text) is abstract
 
 	# Render a code span reading from a buffer.
-	fun add_span_code(v: MarkdownEmitter, buffer: Text, from, to: Int) is abstract
+	fun add_span_code(v: EMITTER, buffer: Text, from, to: Int) is abstract
 
 	# Render a text and escape it.
-	fun append_value(v: MarkdownEmitter, value: Text) is abstract
+	fun append_value(v: EMITTER, value: Text) is abstract
 
 	# Render code text from buffer and escape it.
-	fun append_code(v: MarkdownEmitter, buffer: Text, from, to: Int) is abstract
+	fun append_code(v: EMITTER, buffer: Text, from, to: Int) is abstract
 
 	# Render a character escape.
-	fun escape_char(v: MarkdownEmitter, char: Char) is abstract
+	fun escape_char(v: EMITTER, char: Char) is abstract
 
 	# Render a line break
-	fun add_line_break(v: MarkdownEmitter) is abstract
+	fun add_line_break(v: EMITTER) is abstract
 
 	# Generate a new html valid id from a `String`.
 	fun strip_id(txt: String): String is abstract
@@ -447,7 +758,11 @@ class HTMLDecorator
 	end
 
 	redef fun add_code(v, block) do
-		v.add "<pre><code>"
+		if block isa BlockFence and block.meta != null then
+			v.add "<pre class=\"{block.meta.to_s}\"><code>"
+		else
+			v.add "<pre><code>"
+		end
 		v.emit_in block
 		v.add "</code></pre>\n"
 	end
@@ -488,10 +803,10 @@ class HTMLDecorator
 		v.add "</strong>"
 	end
 
-	redef fun add_super(v, text) do
-		v.add "<sup>"
+	redef fun add_strike(v, text) do
+		v.add "<del>"
 		v.add text
-		v.add "</sup>"
+		v.add "</del>"
 	end
 
 	redef fun add_image(v, link, name, comment) do
@@ -603,9 +918,36 @@ class HTMLDecorator
 	private var allowed_id_chars: Array[Char] = ['-', '_', ':', '.']
 end
 
+# Location in a Markdown input.
+class MDLocation
+
+	# Starting line number (starting from 1).
+	var line_start: Int
+
+	# Starting column number (starting from 1).
+	var column_start: Int
+
+	# Stopping line number (starting from 1).
+	var line_end: Int
+
+	# Stopping column number (starting from 1).
+	var column_end: Int
+
+	redef fun to_s do return "{line_start},{column_start}--{line_end},{column_end}"
+
+	# Return a copy of `self`.
+	fun copy: MDLocation do
+		return new MDLocation(line_start, column_start, line_end, column_end)
+	end
+end
+
 # A block of markdown lines.
 # A `MDBlock` can contains lines and/or sub-blocks.
 class MDBlock
+
+	# Position of `self` in the input.
+	var location: MDLocation
+
 	# Kind of block.
 	# See `Block`.
 	var kind: Block = new BlockNone(self) is writable
@@ -658,7 +1000,14 @@ class MDBlock
 
 	# Split `self` creating a new sub-block having `line` has `last_line`.
 	fun split(line: MDLine): MDBlock do
-		var block = new MDBlock
+		# location for new block
+		var new_loc = new MDLocation(
+			first_line.location.line_start,
+			first_line.location.column_start,
+			line.location.line_end,
+			line.location.column_end)
+		# create block
+		var block = new MDBlock(new_loc)
 		block.first_line = first_line
 		block.last_line = line
 		first_line = line.next
@@ -667,6 +1016,9 @@ class MDBlock
 			last_line = null
 		else
 			first_line.prev = null
+			# update current block loc
+			location.line_start = first_line.location.line_start
+			location.column_start = first_line.location.column_start
 		end
 		if first_block == null then
 			first_block = block
@@ -746,7 +1098,7 @@ class MDBlock
 		var line = first_line
 		while line != null do
 			if not line.is_empty then
-				var kind = line.kind(v)
+				var kind = v.line_kind(line)
 				if kind isa LineList then
 					line.value = kind.extract_value(line)
 				else
@@ -815,10 +1167,32 @@ abstract class Block
 	fun emit_blocks(v: MarkdownEmitter) do
 		var block = self.block.first_block
 		while block != null do
+			v.push_loc(block.location)
 			block.kind.emit(v)
+			v.pop_loc
 			block = block.next
 		end
 	end
+
+	# The raw content of the block as a multi-line string.
+	fun raw_content: String do
+		var infence = self isa BlockFence
+		var text = new FlatBuffer
+		var line = self.block.first_line
+		while line != null do
+			if not line.is_empty then
+				var str = line.value
+				if not infence and str.has_prefix("    ") then
+					text.append str.substring(4, str.length - line.trailing)
+				else
+					text.append str
+				end
+			end
+			text.append "\n"
+			line = line.next
+		end
+		return text.write_to_string
+	end
 end
 
 # A block without any markdown specificities.
@@ -859,13 +1233,21 @@ end
 class BlockCode
 	super Block
 
+	# Any string found after fence token.
+	var meta: nullable Text
+
+	# Number of char to skip at the beginning of the line.
+	#
+	# Block code lines start at 4 spaces.
+	protected var line_start = 4
+
 	redef fun emit(v) do v.decorator.add_code(v, self)
 
 	redef fun emit_lines(v) do
 		var line = block.first_line
 		while line != null do
 			if not line.is_empty then
-				v.decorator.append_code(v, line.value, 4, line.value.length)
+				v.decorator.append_code(v, line.value, line_start, line.value.length)
 			end
 			v.addn
 			line = line.next
@@ -879,13 +1261,24 @@ end
 # this class is only used for typing purposes.
 class BlockFence
 	super BlockCode
+
+	# Fence code lines start at 0 spaces.
+	redef var line_start = 0
 end
 
 # A markdown headline.
 class BlockHeadline
 	super Block
 
-	redef fun emit(v) do v.decorator.add_headline(v, self)
+	redef fun emit(v) do
+		var loc = block.location.copy
+		loc.column_start += start
+		v.push_loc(loc)
+		v.decorator.add_headline(v, self)
+		v.pop_loc
+	end
+
+	private var start = 0
 
 	# Depth of the headline used to determine the headline level.
 	var depth = 0
@@ -914,6 +1307,7 @@ class BlockHeadline
 			line.leading = 0
 			line.trailing = 0
 		end
+		self.start = start
 		depth = level.min(6)
 	end
 end
@@ -935,7 +1329,7 @@ abstract class BlockList
 		var line = block.first_line
 		line = line.next
 		while line != null do
-			var t = line.kind(v)
+			var t = v.line_kind(line)
 			if t isa LineList or
 			   (not line.is_empty and (line.prev_empty and line.leading == 0 and
 			   not (t isa LineList))) then
@@ -1028,6 +1422,9 @@ end
 # A markdown line.
 class MDLine
 
+	# Location of `self` in the original input.
+	var location: MDLocation
+
 	# Text contained in this line.
 	var value: String is writable
 
@@ -1047,8 +1444,8 @@ class MDLine
 	# Is the next line empty?
 	var next_empty: Bool = false is writable
 
-	init(value: String) do
-		self.value = value
+	# Initialize a new MDLine from its string value
+	init do
 		self.leading = process_leading
 		if leading != value.length then
 			self.is_empty = false
@@ -1066,57 +1463,6 @@ class MDLine
 		if next != null then next.prev_empty = true
 	end
 
-	# The type of line.
-	# see `md_line_*`
-	fun kind(v: MarkdownProcessor): Line do
-		var value = self.value
-		if is_empty then return new LineEmpty
-		if leading > 3 then return new LineCode
-		if value[leading] == '#' then return new LineHeadline
-		if value[leading] == '>' then return new LineBlockquote
-
-		if value.length - leading - trailing > 2 then
-			if value[leading] == '`' and count_chars_start('`') >= 3 then
-				return new LineFence
-			end
-			if value[leading] == '~' and count_chars_start('~') >= 3 then
-				return new LineFence
-			end
-		end
-
-		if value.length - leading - trailing > 2 and
-		   (value[leading] == '*' or value[leading] == '-' or value[leading] == '_') then
-		   if count_chars(value[leading]) >= 3 then
-				return new LineHR
-		   end
-		end
-
-		if value.length - leading >= 2 and value[leading + 1] == ' ' then
-			var c = value[leading]
-			if c == '*' or c == '-' or c == '+' then return new LineUList
-		end
-
-		if value.length - leading >= 3 and value[leading].is_digit then
-			var i = leading + 1
-			while i < value.length and value[i].is_digit do i += 1
-			if i + 1 < value.length and value[i] == '.' and value[i + 1] == ' ' then
-				return new LineOList
-			end
-		end
-
-		if value[leading] == '<' and check_html then return new LineXML
-
-		if next != null and not next.is_empty then
-			if next.count_chars('=') > 0 then
-				return new LineHeadline1
-			end
-			if next.count_chars('-') > 0 then
-				return new LineHeadline2
-			end
-		end
-		return new LineOther
-	end
-
 	# Number or leading spaces on this line.
 	var leading: Int = 0 is writable
 
@@ -1304,11 +1650,11 @@ class LineOther
 		# go to block end
 		var was_empty = line.prev_empty
 		while line != null and not line.is_empty do
-			var t = line.kind(v)
-			if v.in_list and t isa LineList then
+			var t = v.line_kind(line)
+			if (v.in_list or v.ext_mode) and t isa LineList then
 				break
 			end
-			if t isa LineCode or t isa LineFence then
+			if v.ext_mode and (t isa LineCode or t isa LineFence) then
 				break
 			end
 			if t isa LineHeadline or t isa LineHeadline1 or t isa LineHeadline2 or
@@ -1318,7 +1664,6 @@ class LineOther
 			line = line.next
 		end
 		# build block
-		var bk: Block
 		if line != null and not line.is_empty then
 			var block = v.current_block.split(line.prev.as(not null))
 			if v.in_list and not was_empty then
@@ -1352,7 +1697,7 @@ class LineCode
 	redef fun process(v) do
 		var line = v.current_line
 		# lookup block end
-		while line != null and (line.is_empty or line.kind(v) isa LineCode) do
+		while line != null and (line.is_empty or v.line_kind(line) isa LineCode) do
 			line = line.next
 		end
 		# split at block end line
@@ -1393,7 +1738,7 @@ class LineBlockquote
 		while line != null do
 			if not line.is_empty and (line.prev_empty and
 			   line.leading == 0 and
-			   not line.kind(v) isa LineBlockquote) then break
+			   not v.line_kind(line) isa LineBlockquote) then break
 			line = line.next
 		end
 		# build sub block
@@ -1435,7 +1780,7 @@ class LineFence
 		# go to fence end
 		var line = v.current_line.next
 		while line != null do
-			if line.kind(v) isa LineFence then break
+			if v.line_kind(line) isa LineFence then break
 			line = line.next
 		end
 		if line != null then
@@ -1448,9 +1793,12 @@ class LineFence
 		else
 			block = v.current_block.split(v.current_block.last_line.as(not null))
 		end
-		block.kind = new BlockFence(block)
+		block.remove_surrounding_empty_lines
+		var meta = block.first_line.value.meta_from_fence
+		block.kind = new BlockFence(block, meta)
 		block.first_line.clear
-		if block.last_line.kind(v) isa LineFence then
+		var last = block.last_line
+		if last != null and v.line_kind(last) isa LineFence then
 			block.last_line.clear
 		end
 		block.remove_surrounding_empty_lines
@@ -1515,14 +1863,14 @@ end
 
 # A markdown list line.
 # Mainly used to factorize code between ordered and unordered lists.
-class LineList
+abstract class LineList
 	super Line
 
 	redef fun process(v) do
 		var line = v.current_line
 		# go to list end
 		while line != null do
-			var t = line.kind(v)
+			var t = v.line_kind(line)
 			if not line.is_empty and (line.prev_empty and line.leading == 0 and
 			   not t isa LineList) then break
 			line = line.next
@@ -1555,6 +1903,7 @@ class LineList
 	# Create a new block kind based on this line.
 	protected fun block_kind(block: MDBlock): BlockList is abstract
 
+	# Extract string value from `MDLine`.
 	protected fun extract_value(line: MDLine): String is abstract
 end
 
@@ -1584,14 +1933,17 @@ end
 # Some tokens have a specific markup behaviour that is handled here.
 abstract class Token
 
-	# Position of `self` in markdown input.
+	# Location of `self` in the original input.
+	var location: MDLocation
+
+	# Position of `self` in input independant from lines.
 	var pos: Int
 
 	# Character found at `pos` in the markdown input.
 	var char: Char
 
 	# Output that token using `MarkdownEmitter::decorator`.
-	fun emit(v: MarkdownEmitter) do v.addc char
+	fun emit(v: MarkdownEmitter) do v.decorator.add_char(v, char)
 end
 
 # A token without a specific meaning.
@@ -1660,7 +2012,7 @@ abstract class TokenCode
 
 	redef fun emit(v) do
 		var a = pos + next_pos + 1
-		var b = v.current_text.find_token(a, self)
+		var b = v.processor.find_token(v.current_text.as(not null), a, self)
 		if b > 0 then
 			v.current_pos = b + next_pos
 			while a < b and v.current_text[a] == ' ' do a += 1
@@ -1775,6 +2127,7 @@ abstract class TokenLinkOrImage
 					if pos == -1 then return -1
 				end
 			end
+			if pos < start then return -1
 			if md[pos] != ')' then return -1
 		else if md[pos] == '[' then
 			pos += 1
@@ -1794,7 +2147,7 @@ abstract class TokenLinkOrImage
 				comment = lr.title
 			end
 		else
-		var tid = name.write_to_string.replace("\n", " ").to_lower
+			var tid = name.write_to_string.replace("\n", " ").to_lower
 			if v.processor.link_refs.has_key(tid) then
 				var lr = v.processor.link_refs[tid]
 				link = lr.link
@@ -1933,17 +2286,19 @@ class TokenEscape
 	end
 end
 
-# A markdown super token.
-class TokenSuper
+# A markdown strike token.
+#
+# Extended mode only (see `MarkdownProcessor::ext_mode`)
+class TokenStrike
 	super Token
 
 	redef fun emit(v) do
 		var tmp = v.push_buffer
-		var b = v.emit_text_until(v.current_text.as(not null), pos + 1, self)
+		var b = v.emit_text_until(v.current_text.as(not null), pos + 2, self)
 		v.pop_buffer
 		if b > 0 then
-			v.decorator.add_super(v, tmp)
-			v.current_pos = b
+			v.decorator.add_strike(v, tmp)
+			v.current_pos = b + 1
 		else
 			v.addc char
 		end
@@ -1952,108 +2307,6 @@ end
 
 redef class Text
 
-	# Get the token kind at `pos`.
-	private fun token_at(pos: Int): Token do
-		var c0: Char
-		var c1: Char
-		var c2: Char
-		var c3: Char
-
-		if pos > 0 then
-			c0 = self[pos - 1]
-		else
-			c0 = ' '
-		end
-		var c = self[pos]
-
-		if pos + 1 < length then
-			c1 = self[pos + 1]
-		else
-			c1 = ' '
-		end
-		if pos + 2 < length then
-			c2 = self[pos + 2]
-		else
-			c2 = ' '
-		end
-		if pos + 3 < length then
-			c3 = self[pos + 3]
-		else
-			c3 = ' '
-		end
-
-		if c == '*' then
-			if c1 == '*' then
-				if c0 != ' ' or c2 != ' ' then
-					return new TokenStrongStar(pos, c)
-				else
-					return new TokenEmStar(pos, c)
-				end
-			end
-			if c0 != ' ' or c1 != ' ' then
-				return new TokenEmStar(pos, c)
-			else
-				return new TokenNone(pos, c)
-			end
-		else if c == '_' then
-			if c1 == '_' then
-				if c0 != ' ' or c2 != ' 'then
-					return new TokenStrongUnderscore(pos, c)
-				else
-					return new TokenEmUnderscore(pos, c)
-				end
-			end
-			if c0 != ' ' or c1 != ' ' then
-				return new TokenEmUnderscore(pos, c)
-			else
-				return new TokenNone(pos, c)
-			end
-		else if c == '!' then
-			if c1 == '[' then return new TokenImage(pos, c)
-			return new TokenNone(pos, c)
-		else if c == '[' then
-			return new TokenLink(pos, c)
-		else if c == ']' then
-			return new TokenNone(pos, c)
-		else if c == '`' then
-			if c1 == '`' then
-				return new TokenCodeDouble(pos, c)
-			else
-				return new TokenCodeSingle(pos, c)
-			end
-		else if c == '\\' then
-			if c1 == '\\' or c1 == '[' or c1 == ']' or c1 == '(' or c1 == ')' or c1 == '{' or c1 == '}' or c1 == '#' or c1 == '"' or c1 == '\'' or c1 == '.' or c1 == '<' or c1 == '>' or c1 == '*' or c1 == '+' or c1 == '-' or c1 == '_' or c1 == '!' or c1 == '`' or c1 == '~' or c1 == '^' then
-				return new TokenEscape(pos, c)
-			else
-				return new TokenNone(pos, c)
-			end
-		else if c == '<' then
-			return new TokenHTML(pos, c)
-		else if c == '&' then
-			return new TokenEntity(pos, c)
-		else if c == '^' then
-			if c0 == '^' or c1 == '^' then
-				return new TokenNone(pos, c)
-			else
-				return new TokenSuper(pos, c)
-			end
-		else
-			return new TokenNone(pos, c)
-		end
-	end
-
-	# Find the position of a `token` in `self`.
-	private fun find_token(start: Int, token: Token): Int do
-		var pos = start
-		while pos < length do
-			if token_at(pos).is_same_type(token) then
-				return pos
-			end
-			pos += 1
-		end
-		return -1
-	end
-
 	# Get the position of the next non-space character.
 	private fun skip_spaces(start: Int): Int do
 		var pos = start
@@ -2160,6 +2413,7 @@ redef class Text
 	# Safe mode can be activated to limit reading to valid xml.
 	private fun read_xml(out: FlatBuffer, start: Int, safe_mode: Bool): Int do
 		var pos = 0
+		var is_valid = true
 		var is_close_tag = false
 		if start + 1 >= length then return -1
 		if self[start + 1] == '/' then
@@ -2177,7 +2431,11 @@ redef class Text
 			pos = read_xml_until(tmp, pos, ' ', '/', '>')
 			if pos == -1 then return -1
 			var tag = tmp.write_to_string.trim.to_lower
-			if tag.is_html_unsafe then
+			if not tag.is_valid_html_tag then
+				out.append "&lt;"
+				pos = -1
+			else if tag.is_html_unsafe then
+				is_valid = false
 				out.append "&lt;"
 				if is_close_tag then out.add '/'
 				out.append tmp
@@ -2200,7 +2458,11 @@ redef class Text
 			if pos == -1 then return -1
 		end
 		if self[pos] == '>' then
-			out.add '>'
+			if is_valid then
+				out.add '>'
+			else
+				out.append "&gt;"
+			end
 			return pos
 		end
 		return -1
@@ -2272,6 +2534,14 @@ redef class Text
 		return tpl.write_to_string.to_lower
 	end
 
+	private fun is_valid_html_tag: Bool do
+		if is_empty then return false
+		for c in self do
+			if not c.is_alpha then return false
+		end
+		return true
+	end
+
 	# Read and escape the markdown contained in `self`.
 	private fun escape(out: FlatBuffer, c: Char, pos: Int): Int do
 		if c == '\\' or c == '[' or c == ']' or c == '(' or c == ')' or c == '{' or
@@ -2285,6 +2555,17 @@ redef class Text
 		return pos
 	end
 
+	# Extract string found at end of fence opening.
+	private fun meta_from_fence: nullable Text do
+		for i in [0..chars.length[ do
+			var c = chars[i]
+			if c != ' ' and c != '`' and c != '~' then
+				return substring_from(i).trim
+			end
+		end
+		return null
+	end
+
 	# Is `self` an unsafe HTML element?
 	private fun is_html_unsafe: Bool do return html_unsafe_tags.has(self.write_to_string)
 
@@ -2308,7 +2589,7 @@ redef class String
 	#    var md = "**Hello World!**"
 	#    var html = md.md_to_html
 	#    assert html == "<p><strong>Hello World!</strong></p>\n"
-	fun md_to_html: Streamable do
+	fun md_to_html: Writable do
 		var processor = new MarkdownProcessor
 		return processor.process(self)
 	end