From: Jean Privat <jean@pryen.org>
Date: Thu, 10 Sep 2015 18:53:56 +0000 (-0400)
Subject: Merge: nitiwiki: external highlighter
X-Git-Tag: v0.7.8~32
X-Git-Url: http://nitlanguage.org?hp=94353a4a0f06dc3eb8181cb0ee4630f234a343d0

Merge: nitiwiki: external highlighter

This is a cleaned up version of what is currently doing the code highlighting and the table in the deployed http://nitlanguage.org

The basic idea here is to delegate the job to some other program. The program has to be configured in the config.ini under the key `wiki.highlighter`

This approach, while not pure nit, makes it quite easy to use whatever highlighter program the user prefer with whatever options (instead of hard-coding a specific external program in the nit source code).

An advanced way is to delegate to a shell script that checks arguments then dispatches to a real program.

ping maintainer: @Morriar

Pull-Request: #1701
Reviewed-by: Lucas Bajolet <r4pass@hotmail.com>
---

diff --git a/contrib/nitiwiki/examples/nitiwiki/config.ini b/contrib/nitiwiki/examples/nitiwiki/config.ini
index 315462d..bc96f38 100644
--- a/contrib/nitiwiki/examples/nitiwiki/config.ini
+++ b/contrib/nitiwiki/examples/nitiwiki/config.ini
@@ -3,3 +3,4 @@ wiki.desc=proudly powered by nit
 wiki.logo=assets/logo.png
 wiki.root_dir=.
 wiki.rsync_dir=moz-code.org:nitiwiki/
+wiki.highlighter=./highlighter.sh "$1"
diff --git a/contrib/nitiwiki/examples/nitiwiki/highlighter.sh b/contrib/nitiwiki/examples/nitiwiki/highlighter.sh
new file mode 100755
index 0000000..57aaf45
--- /dev/null
+++ b/contrib/nitiwiki/examples/nitiwiki/highlighter.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Example of an external highlighter that dispatch on various command
+# according to the argument
+
+# meta is the argument
+meta=$1
+
+# raw is a synonym of txt
+test "$meta" = "raw" && meta=txt
+
+# if `pandoc` then process through the `pandoc` command.
+if test "$meta" = "pandoc"; then
+	exec pandoc -t html
+fi
+
+# Else, try `highlight`
+highlight --fragment -S "$meta" --inline-css --enclose-pre ||
+	# Or `source-highlight`
+	source-highlight -s "$meta"
+out=$?
+exit $out
diff --git a/contrib/nitiwiki/examples/nitiwiki/pages/block.md b/contrib/nitiwiki/examples/nitiwiki/pages/block.md
new file mode 100644
index 0000000..d0daaba
--- /dev/null
+++ b/contrib/nitiwiki/examples/nitiwiki/pages/block.md
@@ -0,0 +1,36 @@
+# Test of code highlighting
+
+A basic block.
+
+    print(["hello", "world"].join(", "))
+
+A fenced block.
+
+~~~
+print(["hello", "world"].join(", "))
+~~~
+
+A fenced block with metainfo, so it can be highlighted with an external tool.
+
+~~~html
+<table class="hello"><tr>
+<td>Hello</td>
+<td>World</td>
+</tr></table>
+~~~
+
+A special block where the rendering is delegated to pandoc.
+For instance, to render tables.
+
+~~~pandoc
+what     how
+------ -----
+hello     10
+world   9001
+~~~
+
+This try some exploit
+~~~raw'"; echo pwned >&2 #
+Hello<br/>
+World
+~~~
diff --git a/contrib/nitiwiki/src/markdown_highlight.nit b/contrib/nitiwiki/src/markdown_highlight.nit
new file mode 100644
index 0000000..08751b0
--- /dev/null
+++ b/contrib/nitiwiki/src/markdown_highlight.nit
@@ -0,0 +1,99 @@
+# This file is part of NIT ( http://www.nitlanguage.org ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Extends the wiki with an external highlighter (or any processor) for code blocks
+module markdown_highlight
+import wiki_links
+
+redef class WikiConfig
+	# External highlighter command called to process block code.
+	#
+	# * key: `wiki.highlighter`
+	# * default: empty string, that means no external highlighter
+	# * example: `highlight --fragment -S "$1" --inline-css --enclose-pre`
+	#
+	# The external highlighter is a shell command invoked with `sh -c`.
+	# The meta information of the fence is passed as `$1`.
+	# *Important*: `$1` is given as is, thus is tainted. You SHOULD protect it with quotes in the command.
+	#
+	# By default, the highlighter is only called on fenced block code with a meta information.
+	# See `wiki.highlighter.default` to force the invocation of the highlighter on any code block.
+	#
+	# The output of the command will be inserted as is in the generated document.
+	# Therefore it is expected that the command returns a valid, complete and balanced HTML fragment.
+	# If the highlighter returns nothing (empty output), the internal rendering is used as a fall-back
+	# (as if the option was not set).
+	#
+	# Advanced usages can invoke a custom shell script instead of a standard command to
+	# check the argument, filter it, dispatch to various advanced commands, implement ad-hoc behaviors, etc.
+	var highlighter: String is lazy do
+		return value_or_default("wiki.highlighter", "")
+	end
+
+	# Default meta (i.e. language) to use to call the external highlighter.
+	#
+	# * key: `wiki.highlighter.default`
+	# * default: empty string, that means no default meta information.
+	# * example: `nit`
+	#
+	# When set, this configuration forces the external highlighter (see `wiki.highlighter`)
+	# to be called also on basic code block (with the indentation) and plain fenced code
+	# blocks (without meta information).
+	#
+	# The value is used as the `$1` argument of the configured highlighter command.
+	#
+	# Note: has no effect if `wiki.highlighter` is not set.
+	var highlighter_default: String is lazy do
+		return value_or_default("wiki.highlighter.default", "")
+	end
+end
+
+redef class NitiwikiDecorator
+	# Extends special cases for meta in fences
+	redef fun add_code(v, block) do
+		var highlighter = wiki.config.highlighter
+
+		# No highlighter, then defaults
+		if highlighter.is_empty then
+			super
+			return
+		end
+
+		var code = block.raw_content
+		var meta = block.meta or else wiki.config.highlighter_default
+
+		# No meta nor forced meta, then defaults
+		if meta.is_empty then
+			super
+			return
+		end
+
+		# Execute the command
+		wiki.message("Executing `{highlighter}` `{meta}` (in {context.src_path.as(not null)})", 2)
+		var proc = new ProcessDuplex("sh", "-c", highlighter, "", meta.to_s)
+		var res = proc.write_and_read(code)
+		if proc.status != 0 then
+			wiki.message("Warning: `{highlighter}` `{meta}` returned {proc.status} (in {context.src_path.as(not null)})", 0)
+		end
+
+		# Check the result
+		if res.is_empty then
+			# No result, then defaults
+			wiki.message("  `{highlighter}` produced nothing, process internally instead (in {context.src_path.as(not null)})", 2)
+			super
+			return
+		end
+		v.add(res)
+	end
+end
diff --git a/contrib/nitiwiki/src/nitiwiki.nit b/contrib/nitiwiki/src/nitiwiki.nit
index 71ab129..c5c6ebe 100644
--- a/contrib/nitiwiki/src/nitiwiki.nit
+++ b/contrib/nitiwiki/src/nitiwiki.nit
@@ -16,6 +16,7 @@
 module nitiwiki
 
 import wiki_html
+import markdown_highlight
 
 # Locate nit directory
 private fun compute_nit_dir(opt_nit_dir: OptionString): String do
diff --git a/contrib/nitiwiki/src/wiki_base.nit b/contrib/nitiwiki/src/wiki_base.nit
index 5f093a0..70cf35f 100644
--- a/contrib/nitiwiki/src/wiki_base.nit
+++ b/contrib/nitiwiki/src/wiki_base.nit
@@ -615,7 +615,7 @@ class WikiConfig
 	super ConfigTree
 
 	# Returns the config value at `key` or return `default` if no key was found.
-	private fun value_or_default(key: String, default: String): String do
+	protected fun value_or_default(key: String, default: String): String do
 		return self[key] or else default
 	end
 
diff --git a/contrib/nitiwiki/src/wiki_links.nit b/contrib/nitiwiki/src/wiki_links.nit
index 340ec74..efb9f78 100644
--- a/contrib/nitiwiki/src/wiki_links.nit
+++ b/contrib/nitiwiki/src/wiki_links.nit
@@ -214,7 +214,8 @@ class NitiwikiMdProcessor
 	end
 end
 
-private class NitiwikiDecorator
+# The decorator associated to `MarkdownProcessor`.
+class NitiwikiDecorator
 	super HTMLDecorator
 
 	# Wiki used to resolve links.
diff --git a/lib/markdown/markdown.nit b/lib/markdown/markdown.nit
index 1bb8815..b9e2f6a 100644
--- a/lib/markdown/markdown.nit
+++ b/lib/markdown/markdown.nit
@@ -758,8 +758,11 @@ class HTMLDecorator
 	end
 
 	redef fun add_code(v, block) do
-		if block isa BlockFence and block.meta != null then
-			v.add "<pre class=\"{block.meta.to_s}\"><code>"
+		var meta = block.meta
+		if meta != null then
+			v.add "<pre class=\""
+			append_value(v, meta)
+			v.add "\"><code>"
 		else
 			v.add "<pre><code>"
 		end
@@ -1173,6 +1176,26 @@ abstract class Block
 			block = block.next
 		end
 	end
+
+	# The raw content of the block as a multi-line string.
+	fun raw_content: String do
+		var infence = self isa BlockFence
+		var text = new FlatBuffer
+		var line = self.block.first_line
+		while line != null do
+			if not line.is_empty then
+				var str = line.value
+				if not infence and str.has_prefix("    ") then
+					text.append str.substring(4, str.length - line.trailing)
+				else
+					text.append str
+				end
+			end
+			text.append "\n"
+			line = line.next
+		end
+		return text.write_to_string
+	end
 end
 
 # A block without any markdown specificities.
@@ -1213,6 +1236,9 @@ end
 class BlockCode
 	super Block
 
+	# Any string found after fence token.
+	var meta: nullable Text
+
 	# Number of char to skip at the beginning of the line.
 	#
 	# Block code lines start at 4 spaces.
@@ -1239,9 +1265,6 @@ end
 class BlockFence
 	super BlockCode
 
-	# Any string found after fence token.
-	var meta: nullable Text
-
 	# Fence code lines start at 0 spaces.
 	redef var line_start = 0
 end
diff --git a/src/doc/doc_down.nit b/src/doc/doc_down.nit
index 5284dc1..e04abb9 100644
--- a/src/doc/doc_down.nit
+++ b/src/doc/doc_down.nit
@@ -99,10 +99,8 @@ private class NitdocDecorator
 	var toolcontext = new ToolContext
 
 	redef fun add_code(v, block) do
-		var meta = "nit"
-		if block isa BlockFence and block.meta != null then
-			meta = block.meta.to_s
-		end
+		var meta = block.meta or else "nit"
+
 		# Do not try to highlight non-nit code.
 		if meta != "nit" and meta != "nitish" then
 			v.add "<pre class=\"{meta}\"><code>"
@@ -111,7 +109,7 @@ private class NitdocDecorator
 			return
 		end
 		# Try to parse code
-		var code = code_from_block(block)
+		var code = block.raw_content
 		var ast = toolcontext.parse_something(code)
 		if ast isa AError then
 			v.add "<pre class=\"{meta}\"><code>"
@@ -150,25 +148,6 @@ private class NitdocDecorator
 		for i in [from..to[ do out.add buffer[i]
 		return out.write_to_string
 	end
-
-	fun code_from_block(block: BlockCode): String do
-		var infence = block isa BlockFence
-		var text = new FlatBuffer
-		var line = block.block.first_line
-		while line != null do
-			if not line.is_empty then
-				var str = line.value
-				if not infence and str.has_prefix("    ") then
-					text.append str.substring(4, str.length - line.trailing)
-				else
-					text.append str
-				end
-			end
-			text.append "\n"
-			line = line.next
-		end
-		return text.write_to_string
-	end
 end
 
 # Decorator for span elements.
diff --git a/src/testing/testing_doc.nit b/src/testing/testing_doc.nit
index 82518da..6bd8f0b 100644
--- a/src/testing/testing_doc.nit
+++ b/src/testing/testing_doc.nit
@@ -290,11 +290,8 @@ private class NitunitDecorator
 	var executor: NitUnitExecutor
 
 	redef fun add_code(v, block) do
-		var code = code_from_block(block)
-		var meta = "nit"
-		if block isa BlockFence and block.meta != null then
-			meta = block.meta.to_s
-		end
+		var code = block.raw_content
+		var meta = block.meta or else "nit"
 		# Do not try to test non-nit code.
 		if meta != "nit" then return
 		# Try to parse code blocks
@@ -321,26 +318,6 @@ private class NitunitDecorator
 		# Add it to the file
 		executor.blocks.last.append code
 	end
-
-	# Extracts code as String from a `BlockCode`.
-	fun code_from_block(block: BlockCode): String do
-		var infence = block isa BlockFence
-		var text = new FlatBuffer
-		var line = block.block.first_line
-		while line != null do
-			if not line.is_empty then
-				var str = line.value
-				if not infence and str.has_prefix("    ") then
-					text.append str.substring(4, str.length - line.trailing)
-				else
-					text.append str
-				end
-			end
-			text.append "\n"
-			line = line.next
-		end
-		return text.write_to_string
-	end
 end
 
 # A unit-test to run
diff --git a/tests/nitiwiki.args b/tests/nitiwiki.args
index ae082af..1ca2cf0 100644
--- a/tests/nitiwiki.args
+++ b/tests/nitiwiki.args
@@ -1,2 +1,2 @@
-nitiwiki --config ../contrib/nitiwiki/tests/wiki1/config2.ini --clean --status
-nitiwiki --config ../contrib/nitiwiki/tests/wiki1/config2.ini --clean --render -v
+--config ../contrib/nitiwiki/tests/wiki1/config2.ini --clean --status
+--config ../contrib/nitiwiki/tests/wiki1/config2.ini --clean --render -v ; rm -r ../contrib/nitiwiki/tests/wiki1/out/