From 0d6a7b2a960367bc1d0dbe752698d52948470f94 Mon Sep 17 00:00:00 2001
From: Lucas Bajolet <r4pass@hotmail.com>
Date: Tue, 11 Aug 2015 11:02:22 -0400
Subject: [PATCH] lib/standard: Added coder and decoder to streams

Signed-off-by: Lucas Bajolet <r4pass@hotmail.com>
---
 lib/standard/stream.nit |   22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/lib/standard/stream.nit b/lib/standard/stream.nit
index ce6e9b3..02eb965 100644
--- a/lib/standard/stream.nit
+++ b/lib/standard/stream.nit
@@ -14,6 +14,7 @@ module stream
 intrude import text::ropes
 import error
 intrude import bytes
+import codecs
 
 in "C" `{
 	#include <unistd.h>
@@ -43,6 +44,10 @@ end
 # A `Stream` that can be read from
 abstract class Reader
 	super Stream
+
+	# Decoder used to transform input bytes to UTF-8
+	var decoder: Decoder = utf8_decoder is writable
+
 	# Reads a character. Returns `null` on EOF or timeout
 	fun read_char: nullable Char is abstract
 
@@ -168,6 +173,7 @@ abstract class Reader
 	# ~~~
 	fun read_all: String do
 		var s = read_all_bytes
+		if not s.is_utf8 then s = s.clean_utf8
 		var slen = s.length
 		if slen == 0 then return ""
 		var rets = ""
@@ -378,6 +384,9 @@ end
 abstract class Writer
 	super Stream
 
+	# The coder from a nit UTF-8 String to the output file
+	var coder: Coder = utf8_coder is writable
+
 	# Writes bytes from `s`
 	fun write_bytes(s: Bytes) is abstract
 
@@ -533,6 +542,7 @@ abstract class BufferedReader
 
 	redef fun append_line_to(s)
 	do
+		var lb = new Bytes.with_capacity(10)
 		loop
 			# First phase: look for a '\n'
 			var i = _buffer_pos
@@ -551,27 +561,29 @@ abstract class BufferedReader
 
 			# if there is something to append
 			if i > _buffer_pos then
-				# Enlarge the string (if needed)
-				s.enlarge(s.bytelen + i - _buffer_pos)
-
 				# Copy from the buffer to the string
 				var j = _buffer_pos
 				while j < i do
-					s.bytes.add(_buffer[j])
+					lb.add(_buffer[j])
 					j += 1
 				end
 				_buffer_pos = i
 			else
 				assert end_reached
+				s.append lb.to_s
 				return
 			end
 
 			if eol then
 				# so \n is found
+				s.append lb.to_s
 				return
 			else
 				# so \n is not found
-				if end_reached then return
+				if end_reached then
+					s.append lb.to_s
+					return
+				end
 				fill_buffer
 			end
 		end
-- 
1.7.9.5