X-Git-Url: http://nitlanguage.org

diff --git a/lib/core/stream.nit b/lib/core/stream.nit
index d884b62..b0486ef 100644
--- a/lib/core/stream.nit
+++ b/lib/core/stream.nit
@@ -122,14 +122,81 @@ abstract class Reader
 		for i in [0 .. max[ do
 			var b = raw_read_byte
 			if b < 0 then break
-			buf[i] = b.to_b
+			buf[i] = b
 			rd += 1
 		end
 		return rd
 	end
 
 	# Reads a character. Returns `null` on EOF or timeout
-	fun read_char: nullable Char is abstract
+	#
+	# Returns unicode replacement character '�' if an
+	# invalid byte sequence is read.
+	#
+	# `read_char` may block if:
+	#
+	# * No byte could be read from the current buffer
+	# * An incomplete char is partially read, and more bytes are
+	#   required for full decoding.
+	fun read_char: nullable Char do
+		if eof then return null
+		var cod = codec
+		var codet_sz = cod.codet_size
+		var lk = lookahead
+		var llen = lookahead_length
+		if llen < codet_sz then
+			llen += raw_read_bytes(lk.fast_cstring(llen), codet_sz - llen)
+		end
+		if llen < codet_sz then
+			lookahead_length = 0
+			return 0xFFFD.code_point
+		end
+		var ret = cod.is_valid_char(lk, codet_sz)
+		var max_llen = cod.max_lookahead
+		while ret == 1 and llen < max_llen do
+			var rd = raw_read_bytes(lk.fast_cstring(llen), codet_sz)
+			if rd < codet_sz then
+				llen -= codet_sz
+				if llen > 0 then
+					lookahead.lshift(codet_sz, llen, codet_sz)
+				end
+				lookahead_length = llen.max(0)
+				return 0xFFFD.code_point
+			end
+			llen += codet_sz
+			ret = cod.is_valid_char(lk, llen)
+		end
+		if ret == 0 then
+			var c = cod.decode_char(lk)
+			var clen = c.u8char_len
+			llen -= clen
+			if llen > 0 then
+				lookahead.lshift(clen, llen, clen)
+			end
+			lookahead_length = llen
+			return c
+		end
+		if ret == 2 or ret == 1 then
+			llen -= codet_sz
+			if llen > 0 then
+				lookahead.lshift(codet_sz, llen, codet_sz)
+			end
+			lookahead_length = llen
+			return 0xFFFD.code_point
+		end
+		# Should not happen if the decoder works properly
+		var arr = new Array[Object]
+		arr.push "Decoder error: could not decode nor recover from byte sequence ["
+		for i in [0 .. llen[ do
+			arr.push lk[i]
+			arr.push ", "
+		end
+		arr.push "]"
+		var err = new IOError(arr.plain_to_s)
+		err.cause = last_error
+		last_error = err
+		return 0xFFFD.code_point
+	end
 
 	# Reads a byte. Returns a negative value on error
 	fun read_byte: Int do
@@ -148,13 +215,16 @@ abstract class Reader
 
 	# Reads a String of at most `i` length
 	fun read(i: Int): String do
+		assert i >= 0
 		var cs = new CString(i)
 		var rd = read_bytes_to_cstring(cs, i)
+		if rd < 0 then return ""
 		return codec.decode_string(cs, rd)
 	end
 
 	# Reads up to `max` bytes from source
 	fun read_bytes(max: Int): Bytes do
+		assert max >= 0
 		var cs = new CString(max)
 		var rd = read_bytes_to_cstring(cs, max)
 		return new Bytes(cs, rd, max)
@@ -173,7 +243,7 @@ abstract class Reader
 		else
 			lookahead_length = 0
 		end
-		return rd + raw_read_bytes(bytes, max - rd)
+		return rd + raw_read_bytes(bytes.fast_cstring(rd), max - rd)
 	end
 
 	# Read a string until the end of the line.
@@ -345,7 +415,11 @@ abstract class Reader
 
 	# Is there something to read.
 	# This function returns 'false' if there is something to read.
-	fun eof: Bool is abstract
+	fun eof: Bool do
+		if lookahead_length > 0 then return false
+		lookahead_length = raw_read_bytes(lookahead, 1)
+		return lookahead_length <= 0
+	end
 
 	# Read the next sequence of non whitespace characters.
 	#
@@ -410,37 +484,18 @@ end
 # Iterator returned by `Reader::each_line`.
 # See the aforementioned method for details.
 class LineIterator
-	super Iterator[String]
+	super CachedIterator[String]
 
 	# The original stream
 	var stream: Reader
 
-	redef fun is_ok
+	redef fun next_item
 	do
-		var res = not stream.eof
-		if not res and close_on_finish then stream.close
-		return res
-	end
-
-	redef fun item
-	do
-		var line = self.line
-		if line == null then
-			line = stream.read_line
+		if stream.eof then
+			if close_on_finish then stream.close
+			return null
 		end
-		self.line = line
-		return line
-	end
-
-	# The last line read (cache)
-	private var line: nullable String = null
-
-	redef fun next
-	do
-		# force the read
-		if line == null then item
-		# drop the line
-		line = null
+		return stream.read_line
 	end
 
 	# Close the stream when the stream is at the EOF.
@@ -477,7 +532,7 @@ abstract class Writer
 	fun write(s: Text) is abstract
 
 	# Write a single byte
-	fun write_byte(value: Byte) is abstract
+	fun write_byte(value: Int) is abstract
 
 	# Write a single char
 	fun write_char(c: Char) do
@@ -500,17 +555,32 @@ interface Writable
 	# The specific logic it let to the concrete subclasses
 	fun write_to(stream: Writer) is abstract
 
-	# Like `write_to` but return a new String (may be quite large)
+	# Like `write_to` but return a new String (may be quite large).
 	#
-	# This funtionality is anectodical, since the point
-	# of streamable object to to be efficienlty written to a
-	# stream without having to allocate and concatenate strings
+	# This functionality is anecdotal, since the point
+	# of a streamable object is to be efficiently written to a
+	# stream without having to allocate and concatenate strings.
 	fun write_to_string: String
 	do
 		var stream = new StringWriter
 		write_to(stream)
 		return stream.to_s
 	end
+
+	# Like `write_to` but return a new Bytes (may be quite large)
+	#
+	# This functionality is anecdotal, since the point
+	# of a streamable object is to be efficiently written to a
+	# stream without having to allocate and concatenate buffers.
+	#
+	# Nevertheless, you might need this method if you want to know
+	# the byte size of a writable object.
+	fun write_to_bytes: Bytes
+	do
+		var stream = new BytesWriter
+		write_to(stream)
+		return stream.bytes
+	end
 end
 
 redef class Bytes
@@ -525,201 +595,6 @@ redef class Text
 	redef fun write_to(stream) do stream.write(self)
 end
 
-# Input streams with a buffered input for efficiency purposes
-abstract class BufferedReader
-	super Reader
-	redef fun read_char
-	do
-		if last_error != null then return null
-		if eof then
-			last_error = new IOError("Stream has reached eof")
-			return null
-		end
-		# TODO: Fix when supporting UTF-8
-		var c = _buffer[_buffer_pos].to_i.code_point
-		_buffer_pos += 1
-		return c
-	end
-
-	redef fun read_byte
-	do
-		if last_error != null then return -1
-		if eof then
-			last_error = new IOError("Stream has reached eof")
-			return -1
-		end
-		var c = _buffer[_buffer_pos]
-		_buffer_pos += 1
-		return c.to_i
-	end
-
-	# Resets the internal buffer
-	fun buffer_reset do
-		_buffer_length = 0
-		_buffer_pos = 0
-	end
-
-	# Peeks up to `n` bytes in the buffer
-	#
-	# The operation does not consume the buffer
-	#
-	# ~~~nitish
-	# var x = new FileReader.open("File.txt")
-	# assert x.peek(5) == x.read(5)
-	# ~~~
-	fun peek(i: Int): Bytes do
-		if eof then return new Bytes.empty
-		var remsp = _buffer_length - _buffer_pos
-		if i <= remsp then
-			var bf = new Bytes.with_capacity(i)
-			bf.append_ns_from(_buffer, i, _buffer_pos)
-			return bf
-		end
-		var bf = new Bytes.with_capacity(i)
-		bf.append_ns_from(_buffer, remsp, _buffer_pos)
-		_buffer_pos = _buffer_length
-		read_intern(i - bf.length, bf)
-		remsp = _buffer_length - _buffer_pos
-		var full_len = bf.length + remsp
-		if full_len > _buffer_capacity then
-			var c = _buffer_capacity
-			while c < full_len do c = c * 2 + 2
-			_buffer_capacity = c
-		end
-		var nns = new CString(_buffer_capacity)
-		bf.items.copy_to(nns, bf.length, 0, 0)
-		_buffer.copy_to(nns, remsp, _buffer_pos, bf.length)
-		_buffer = nns
-		_buffer_pos = 0
-		_buffer_length = full_len
-		return bf
-	end
-
-	redef fun read_bytes_to_cstring(buf, i)
-	do
-		if last_error != null then return 0
-		var bbf = new Bytes(buf, 0, i)
-		return read_intern(i, bbf)
-	end
-
-	# Fills `buf` with at most `i` bytes read from `self`
-	private fun read_intern(i: Int, buf: Bytes): Int do
-		if eof then return 0
-		var p = _buffer_pos
-		var bufsp = _buffer_length - p
-		if bufsp >= i then
-			_buffer_pos += i
-			buf.append_ns_from(_buffer, i, p)
-			return i
-		end
-		_buffer_pos = _buffer_length
-		var readln = _buffer_length - p
-		buf.append_ns_from(_buffer, readln, p)
-		var rd = read_intern(i - readln, buf)
-		return rd + readln
-	end
-
-	redef fun read_all_bytes
-	do
-		if last_error != null then return new Bytes.empty
-		var s = new Bytes.with_capacity(10)
-		var b = _buffer
-		while not eof do
-			var j = _buffer_pos
-			var k = _buffer_length
-			var rd_sz = k - j
-			s.append_ns_from(b, rd_sz, j)
-			_buffer_pos = k
-			fill_buffer
-		end
-		return s
-	end
-
-	redef fun append_line_to(s)
-	do
-		var lb = new Bytes.with_capacity(10)
-		loop
-			# First phase: look for a '\n'
-			var i = _buffer_pos
-			while i < _buffer_length and _buffer[i] != 0xAu8 do
-				i += 1
-			end
-
-			var eol
-			if i < _buffer_length then
-				assert _buffer[i] == 0xAu8
-				i += 1
-				eol = true
-			else
-				eol = false
-			end
-
-			# if there is something to append
-			if i > _buffer_pos then
-				# Copy from the buffer to the string
-				var j = _buffer_pos
-				while j < i do
-					lb.add(_buffer[j])
-					j += 1
-				end
-				_buffer_pos = i
-			else
-				assert end_reached
-				s.append lb.to_s
-				return
-			end
-
-			if eol then
-				# so \n is found
-				s.append lb.to_s
-				return
-			else
-				# so \n is not found
-				if end_reached then
-					s.append lb.to_s
-					return
-				end
-				fill_buffer
-			end
-		end
-	end
-
-	redef fun eof
-	do
-		if _buffer_pos < _buffer_length then return false
-		if end_reached then return true
-		fill_buffer
-		return _buffer_pos >= _buffer_length and end_reached
-	end
-
-	# The buffer
-	private var buffer: CString = new CString(0)
-
-	# The current position in the buffer
-	private var buffer_pos = 0
-
-	# Length of the current buffer (i.e. nuber of bytes in the buffer)
-	private var buffer_length = 0
-
-	# Capacity of the buffer
-	private var buffer_capacity = 0
-
-	# Fill the buffer
-	protected fun fill_buffer is abstract
-
-	# Has the last fill_buffer reached the end
-	protected fun end_reached: Bool is abstract
-
-	# Allocate a `_buffer` for a given `capacity`.
-	protected fun prepare_buffer(capacity: Int)
-	do
-		_buffer = new CString(capacity)
-		_buffer_pos = 0 # need to read
-		_buffer_length = 0
-		_buffer_capacity = capacity
-	end
-end
-
 # A `Stream` that can be written to and read from
 abstract class Duplex
 	super Reader
@@ -733,7 +608,7 @@ end
 #
 # writer.write "Strings "
 # writer.write_char '&'
-# writer.write_byte 0x20u8
+# writer.write_byte 0x20
 # writer.write_bytes "bytes".to_bytes
 #
 # assert writer.to_s == "\\x53\\x74\\x72\\x69\\x6E\\x67\\x73\\x20\\x26\\x20\\x62\\x79\\x74\\x65\\x73"
@@ -747,12 +622,12 @@ end
 # writer = new BytesWriter
 #
 # # Write just the character first half
-# writer.write_byte 0xC2u8
+# writer.write_byte 0xC2
 # assert writer.to_s == "\\xC2"
 # assert writer.bytes.to_s == "�"
 #
 # # Complete the character
-# writer.write_byte 0xA2u8
+# writer.write_byte 0xA2
 # assert writer.to_s == "\\xC2\\xA2"
 # assert writer.bytes.to_s == "¢"
 # ~~~
@@ -804,7 +679,7 @@ end
 #
 # writer.write "Strings "
 # writer.write_char '&'
-# writer.write_byte 0x20u8
+# writer.write_byte 0x20
 # writer.write_bytes "bytes".to_bytes
 #
 # assert writer.to_s == "Strings & bytes"
@@ -834,17 +709,7 @@ class BytesReader
 	# The current position in `bytes`
 	private var cursor = 0
 
-	redef fun read_char
-	do
-		if cursor >= bytes.length then return null
-
-		var len = bytes.items.length_of_char_at(cursor)
-		var char = bytes.items.char_at(cursor)
-		cursor += len
-		return char
-	end
-
-	redef fun read_byte
+	redef fun raw_read_byte
 	do
 		if cursor >= bytes.length then return -1