# This file is part of NIT ( http://www.nitlanguage.org ).
#
-# Copyright 2004-2008 Jean Privat <jean@pryen.org>
-#
-# This file is free software, which comes along with NIT. This software is
+# This file is free software, which comes along with NIT. This software is
# distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
-# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE. You can modify it is you want, provided this header
+# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. You can modify it is you want, provided this header
# is kept unaltered, and a notification of the changes is added.
-# You are allowed to redistribute it and sell it, alone or is a part of
+# You are allowed to redistribute it and sell it, alone or is a part of
# another product.
# Input and output streams of characters
module stream
-import string
+intrude import ropes
+import error
+intrude import bytes
in "C" `{
#include <unistd.h>
- #include <poll.h>
- #include <errno.h>
#include <string.h>
+ #include <signal.h>
`}
-# Abstract stream class
-interface IOS
+# Any kind of error that could be produced by an operation on Streams
+class IOError
+ super Error
+end
+
+# Any kind of stream to read/write/both to or from a source
+abstract class Stream
+ # Error produced by the file stream
+ #
+ # var ifs = new FileReader.open("donotmakethisfile.binx")
+ # ifs.read_all
+ # ifs.close
+ # assert ifs.last_error != null
+ var last_error: nullable IOError = null
+
# close the stream
fun close is abstract
end
-# Abstract input streams
-interface IStream
- super IOS
- # Read a character. Return its ASCII value, -1 on EOF or timeout
- fun read_char: Int is abstract
+# A `Stream` that can be read from
+abstract class Reader
+ super Stream
+ # Reads a character. Returns `null` on EOF or timeout
+ fun read_char: nullable Char is abstract
+
+ # Reads a byte. Returns `null` on EOF or timeout
+ fun read_byte: nullable Int is abstract
+
+ # Reads a String of at most `i` length
+ fun read(i: Int): String do return read_bytes(i).to_s
# Read at most i bytes
- fun read(i: Int): String
+ fun read_bytes(i: Int): Bytes
do
- var s = new Buffer.with_capacity(i)
+ if last_error != null then return new Bytes.empty
+ var s = new NativeString(i)
+ var buf = new Bytes(s, 0, 0)
while i > 0 and not eof do
- var c = read_char
- if c >= 0 then
- s.add(c.ascii)
+ var c = read_byte
+ if c != null then
+ buf.add c
i -= 1
end
end
- return s.to_s
+ return buf
end
# Read a string until the end of the line.
+ #
+ # The line terminator '\n' and '\r\n', if any, is removed in each line.
+ #
+ # ~~~
+ # var txt = "Hello\n\nWorld\n"
+ # var i = new StringReader(txt)
+ # assert i.read_line == "Hello"
+ # assert i.read_line == ""
+ # assert i.read_line == "World"
+ # assert i.eof
+ # ~~~
+ #
+ # Only LINE FEED (`\n`), CARRIAGE RETURN & LINE FEED (`\r\n`), and
+ # the end or file (EOF) is considered to delimit the end of lines.
+ # CARRIAGE RETURN (`\r`) alone is not used for the end of line.
+ #
+ # ~~~
+ # var txt2 = "Hello\r\n\n\rWorld"
+ # var i2 = new StringReader(txt2)
+ # assert i2.read_line == "Hello"
+ # assert i2.read_line == ""
+ # assert i2.read_line == "\rWorld"
+ # assert i2.eof
+ # ~~~
+ #
+ # NOTE: Use `append_line_to` if the line terminator needs to be preserved.
fun read_line: String
do
- assert not eof
- var s = new Buffer
+ if last_error != null then return ""
+ if eof then return ""
+ var s = new FlatBuffer
append_line_to(s)
- return s.to_s
+ return s.to_s.chomp
end
+ # Read all the lines until the eof.
+ #
+ # The line terminator '\n' and `\r\n` is removed in each line,
+ #
+ # ~~~
+ # var txt = "Hello\n\nWorld\n"
+ # var i = new StringReader(txt)
+ # assert i.read_lines == ["Hello", "", "World"]
+ # ~~~
+ #
+ # This method is more efficient that splitting
+ # the result of `read_all`.
+ #
+ # NOTE: SEE `read_line` for details.
+ fun read_lines: Array[String]
+ do
+ var res = new Array[String]
+ while not eof do
+ res.add read_line
+ end
+ return res
+ end
+
+ # Return an iterator that read each line.
+ #
+ # The line terminator '\n' and `\r\n` is removed in each line,
+ # The line are read with `read_line`. See this method for details.
+ #
+ # ~~~
+ # var txt = "Hello\n\nWorld\n"
+ # var i = new StringReader(txt)
+ # assert i.each_line.to_a == ["Hello", "", "World"]
+ # ~~~
+ #
+ # Unlike `read_lines` that read all lines at the call, `each_line` is lazy.
+ # Therefore, the stream should no be closed until the end of the stream.
+ #
+ # ~~~
+ # i = new StringReader(txt)
+ # var el = i.each_line
+ #
+ # assert el.item == "Hello"
+ # el.next
+ # assert el.item == ""
+ # el.next
+ #
+ # i.close
+ #
+ # assert not el.is_ok
+ # # closed before "world" is read
+ # ~~~
+ fun each_line: LineIterator do return new LineIterator(self)
+
# Read all the stream until the eof.
- fun read_all: String
+ #
+ # The content of the file is returned as a String.
+ #
+ # ~~~
+ # var txt = "Hello\n\nWorld\n"
+ # var i = new StringReader(txt)
+ # assert i.read_all == txt
+ # ~~~
+ fun read_all: String do return read_all_bytes.to_s
+
+ # Read all the stream until the eof.
+ #
+ # The content of the file is returned verbatim.
+ fun read_all_bytes: Bytes
do
- var s = new Buffer
+ if last_error != null then return new Bytes.empty
+ var s = new Bytes.empty
while not eof do
- var c = read_char
- if c >= 0 then s.add(c.ascii)
+ var c = read_byte
+ if c != null then s.add(c)
end
- return s.to_s
+ return s
end
# Read a string until the end of the line and append it to `s`.
+ #
+ # Unlike `read_line` and other related methods,
+ # the line terminator '\n', if any, is preserved in each line.
+ # Use the method `Text::chomp` to safely remove it.
+ #
+ # ~~~
+ # var txt = "Hello\n\nWorld\n"
+ # var i = new StringReader(txt)
+ # var b = new FlatBuffer
+ # i.append_line_to(b)
+ # assert b == "Hello\n"
+ # i.append_line_to(b)
+ # assert b == "Hello\n\n"
+ # i.append_line_to(b)
+ # assert b == txt
+ # assert i.eof
+ # ~~~
+ #
+ # If `\n` is not present at the end of the result, it means that
+ # a non-eol terminated last line was returned.
+ #
+ # ~~~
+ # var i2 = new StringReader("hello")
+ # assert not i2.eof
+ # var b2 = new FlatBuffer
+ # i2.append_line_to(b2)
+ # assert b2 == "hello"
+ # assert i2.eof
+ # ~~~
+ #
+ # NOTE: The single character LINE FEED (`\n`) delimits the end of lines.
+ # Therefore CARRIAGE RETURN & LINE FEED (`\r\n`) is also recognized.
fun append_line_to(s: Buffer)
do
+ if last_error != null then return
loop
var x = read_char
- if x == -1 then
+ if x == null then
if eof then return
else
- var c = x.ascii
- s.chars.push(c)
- if c == '\n' then return
+ s.chars.push(x)
+ if x == '\n' then return
end
end
end
# Is there something to read.
# This function returns 'false' if there is something to read.
fun eof: Bool is abstract
+
+ # Read the next sequence of non whitespace characters.
+ #
+ # Leading whitespace characters are skipped.
+ # The first whitespace character that follows the result is consumed.
+ #
+ # An empty string is returned if the end of the file or an error is encounter.
+ #
+ # ~~~
+ # var w = new StringReader(" Hello, \n\t World!")
+ # assert w.read_word == "Hello,"
+ # assert w.read_char == '\n'.ascii
+ # assert w.read_word == "World!"
+ # assert w.read_word == ""
+ # ~~~
+ #
+ # `Char::is_whitespace` determines what is a whitespace.
+ fun read_word: String
+ do
+ var buf = new FlatBuffer
+ var c = read_nonwhitespace
+ if c != null then
+ buf.add(c)
+ while not eof do
+ c = read_char
+ if c == null then break
+ if c.is_whitespace then break
+ buf.add(c)
+ end
+ end
+ var res = buf.to_s
+ return res
+ end
+
+ # Skip whitespace characters (if any) then return the following non-whitespace character.
+ #
+ # Returns the code point of the character.
+ # Returns `null` on end of file or error.
+ #
+ # In fact, this method works like `read_char` except it skips whitespace.
+ #
+ # ~~~
+ # var w = new StringReader(" \nab\tc")
+ # assert w.read_nonwhitespace == 'a'
+ # assert w.read_nonwhitespace == 'b'
+ # assert w.read_nonwhitespace == 'c'
+ # assert w.read_nonwhitespace == null
+ # ~~~
+ #
+ # `Char::is_whitespace` determines what is a whitespace.
+ fun read_nonwhitespace: nullable Char
+ do
+ var c: nullable Char = null
+ while not eof do
+ c = read_char
+ if c == null or not c.is_whitespace then break
+ end
+ return c
+ end
+end
+
+# Iterator returned by `Reader::each_line`.
+# See the aforementioned method for details.
+class LineIterator
+ super Iterator[String]
+
+ # The original stream
+ var stream: Reader
+
+ redef fun is_ok
+ do
+ var res = not stream.eof
+ if not res and close_on_finish then stream.close
+ return res
+ end
+
+ redef fun item
+ do
+ var line = self.line
+ if line == null then
+ line = stream.read_line
+ end
+ self.line = line
+ return line
+ end
+
+ # The last line read (cache)
+ private var line: nullable String = null
+
+ redef fun next
+ do
+ # force the read
+ if line == null then item
+ # drop the line
+ line = null
+ end
+
+ # Close the stream when the stream is at the EOF.
+ #
+ # Default is false.
+ var close_on_finish = false is writable
+
+ redef fun finish
+ do
+ if close_on_finish then stream.close
+ end
+end
+
+# `Reader` capable of declaring if readable without blocking
+abstract class PollableReader
+ super Reader
+
+ # Is there something to read? (without blocking)
+ fun poll_in: Bool is abstract
+
end
-# Abstract output stream
-interface OStream
- super IOS
+# A `Stream` that can be written to
+abstract class Writer
+ super Stream
+
+ # Writes bytes from `s`
+ fun write_bytes(s: Bytes) is abstract
+
# write a string
- fun write(s: String) is abstract
+ fun write(s: Text) is abstract
+
+ # Write a single byte
+ fun write_byte(value: Int) is abstract
# Can the stream be used to write
fun is_writable: Bool is abstract
end
-# Input streams with a buffer
-abstract class BufferedIStream
- super IStream
+# Things that can be efficienlty written to a `Writer`
+#
+# The point of this interface is to allow the instance to be efficiently
+# written into a `Writer`.
+#
+# Ready-to-save documents usually provide this interface.
+interface Writable
+ # Write itself to a `stream`
+ # The specific logic it let to the concrete subclasses
+ fun write_to(stream: Writer) is abstract
+
+ # Like `write_to` but return a new String (may be quite large)
+ #
+ # This funtionality is anectodical, since the point
+ # of streamable object to to be efficienlty written to a
+ # stream without having to allocate and concatenate strings
+ fun write_to_string: String
+ do
+ var stream = new StringWriter
+ write_to(stream)
+ return stream.to_s
+ end
+end
+
+redef class Text
+ super Writable
+ redef fun write_to(stream) do stream.write(self)
+end
+
+# Input streams with a buffered input for efficiency purposes
+abstract class BufferedReader
+ super Reader
redef fun read_char
do
- assert not eof
- if _buffer_pos >= _buffer.length then
- fill_buffer
+ if last_error != null then return null
+ if eof then
+ last_error = new IOError("Stream has reached eof")
+ return null
end
- if _buffer_pos >= _buffer.length then
- return -1
+ var c = _buffer[_buffer_pos]
+ _buffer_pos += 1
+ return c
+ end
+
+ redef fun read_byte
+ do
+ if last_error != null then return null
+ if eof then
+ last_error = new IOError("Stream has reached eof")
+ return null
end
- var c = _buffer.chars[_buffer_pos]
+ var c = _buffer[_buffer_pos].ascii
_buffer_pos += 1
- return c.ascii
+ return c
end
- redef fun read(i)
+ fun buffer_reset do
+ _buffer_length = 0
+ _buffer_pos = 0
+ end
+
+ # Peeks up to `n` bytes in the buffer
+ #
+ # The operation does not consume the buffer
+ #
+ # ~~~nitish
+ # var x = new FileReader.open("File.txt")
+ # assert x.peek(5) == x.read(5)
+ # ~~~
+ fun peek(i: Int): Bytes do
+ if eof then return new Bytes.empty
+ var remsp = _buffer_length - _buffer_pos
+ if i <= remsp then
+ var bf = new Bytes.with_capacity(i)
+ bf.append_ns_from(_buffer, i, _buffer_pos)
+ return bf
+ end
+ var bf = new Bytes.with_capacity(i)
+ bf.append_ns_from(_buffer, remsp, _buffer_pos)
+ _buffer_pos = _buffer_length
+ read_intern(i - bf.length, bf)
+ remsp = _buffer_length - _buffer_pos
+ var full_len = bf.length + remsp
+ if full_len > _buffer_capacity then
+ var c = _buffer_capacity
+ while c < full_len do c = c * 2 + 2
+ _buffer_capacity = c
+ end
+ var nns = new NativeString(_buffer_capacity)
+ bf.items.copy_to(nns, bf.length, 0, 0)
+ _buffer.copy_to(nns, remsp, _buffer_pos, bf.length)
+ _buffer = nns
+ _buffer_pos = 0
+ _buffer_length = full_len
+ return bf
+ end
+
+ redef fun read_bytes(i)
do
- var s = new Buffer.with_capacity(i)
- var j = _buffer_pos
- var k = _buffer.length
- while i > 0 do
- if j >= k then
- fill_buffer
- if eof then return s.to_s
- j = _buffer_pos
- k = _buffer.length
- end
- while j < k and i > 0 do
- s.add(_buffer.chars[j])
- j += 1
- i -= 1
- end
+ if last_error != null then return new Bytes.empty
+ var buf = new Bytes.with_capacity(i)
+ read_intern(i, buf)
+ return buf
+ end
+
+ # Fills `buf` with at most `i` bytes read from `self`
+ private fun read_intern(i: Int, buf: Bytes): Int do
+ if eof then return 0
+ var p = _buffer_pos
+ var bufsp = _buffer_length - p
+ if bufsp >= i then
+ _buffer_pos += i
+ buf.append_ns_from(_buffer, i, p)
+ return i
end
- _buffer_pos = j
- return s.to_s
+ _buffer_pos = _buffer_length
+ var readln = _buffer_length - p
+ buf.append_ns_from(_buffer, readln, p)
+ var rd = read_intern(i - readln, buf)
+ return rd + readln
end
- redef fun read_all
+ redef fun read_all_bytes
do
- var s = new Buffer
+ if last_error != null then return new Bytes.empty
+ var s = new Bytes.with_capacity(10)
while not eof do
var j = _buffer_pos
- var k = _buffer.length
+ var k = _buffer_length
while j < k do
- s.add(_buffer.chars[j])
+ s.add(_buffer[j].ascii)
j += 1
end
_buffer_pos = j
fill_buffer
end
- return s.to_s
- end
+ return s
+ end
redef fun append_line_to(s)
do
loop
# First phase: look for a '\n'
var i = _buffer_pos
- while i < _buffer.length and _buffer.chars[i] != '\n' do i += 1
+ while i < _buffer_length and _buffer[i] != '\n' do
+ i += 1
+ end
+
+ var eol
+ if i < _buffer_length then
+ assert _buffer[i] == '\n'
+ i += 1
+ eol = true
+ else
+ eol = false
+ end
# if there is something to append
if i > _buffer_pos then
# Copy from the buffer to the string
var j = _buffer_pos
while j < i do
- s.add(_buffer.chars[j])
+ s.add(_buffer[j])
j += 1
end
+ _buffer_pos = i
+ else
+ assert end_reached
+ return
end
- if i < _buffer.length then
- # so \n is in _buffer[i]
- _buffer_pos = i + 1 # skip \n
+ if eol then
+ # so \n is found
return
else
# so \n is not found
- _buffer_pos = i
- if end_reached then
- return
- else
- fill_buffer
- end
+ if end_reached then return
+ fill_buffer
end
end
end
- redef fun eof do return _buffer_pos >= _buffer.length and end_reached
+ redef fun eof
+ do
+ if _buffer_pos < _buffer_length then return false
+ if end_reached then return true
+ fill_buffer
+ return _buffer_pos >= _buffer_length and end_reached
+ end
# The buffer
- var _buffer: nullable Buffer = null
+ private var buffer: NativeString = new NativeString(0)
# The current position in the buffer
- var _buffer_pos: Int = 0
+ private var buffer_pos = 0
+
+ # Length of the current buffer (i.e. nuber of bytes in the buffer)
+ private var buffer_length = 0
+
+ # Capacity of the buffer
+ private var buffer_capacity = 0
# Fill the buffer
protected fun fill_buffer is abstract
- # Is the last fill_buffer reach the end
+ # Has the last fill_buffer reached the end
protected fun end_reached: Bool is abstract
# Allocate a `_buffer` for a given `capacity`.
protected fun prepare_buffer(capacity: Int)
do
- _buffer = new Buffer.with_capacity(capacity)
+ _buffer = new NativeString(capacity)
_buffer_pos = 0 # need to read
+ _buffer_length = 0
+ _buffer_capacity = capacity
end
end
-interface IOStream
- super IStream
- super OStream
+# A `Stream` that can be written to and read from
+abstract class Duplex
+ super Reader
+ super Writer
end
-##############################################################"
-
-abstract class FDStream
- super IOS
- # File description
- var fd: Int
-
- redef fun close do native_close(fd)
+# `Stream` that can be used to write to a `String`
+#
+# Mainly used for compatibility with Writer type and tests.
+class StringWriter
+ super Writer
- private fun native_close(i: Int): Int is extern "stream_FDStream_FDStream_native_close_1"
- private fun native_read_char(i: Int): Int is extern "stream_FDStream_FDStream_native_read_char_1"
- private fun native_read(i: Int, buf: NativeString, len: Int): Int is extern "stream_FDStream_FDStream_native_read_3"
- private fun native_write(i: Int, buf: NativeString, len: Int): Int is extern "stream_FDStream_FDStream_native_write_3"
- private fun native_write_char(i: Int, c: Char): Int is extern "stream_FDStream_FDStream_native_write_char_2"
+ private var content = new Array[String]
+ redef fun to_s do return content.to_s
+ redef fun is_writable do return not closed
- init(fd: Int) do self.fd = fd
-end
+ redef fun write_bytes(b) do
+ content.add(b.to_s)
+ end
-class FDIStream
- super FDStream
- super IStream
- redef var eof: Bool = false
-
- redef fun read_char
+ redef fun write(str)
do
- var nb = native_read_char(fd)
- if nb == -1 then eof = true
- return nb
+ assert not closed
+ content.add(str.to_s)
end
- init(fd: Int) do end
+ # Is the stream closed?
+ protected var closed = false
+
+ redef fun close do closed = true
end
-class FDOStream
- super FDStream
- super OStream
- redef var is_writable: Bool
+# `Stream` used to read from a `String`
+#
+# Mainly used for compatibility with Reader type and tests.
+class StringReader
+ super Reader
- redef fun write(s)
- do
- var nb = native_write(fd, s.to_cstring, s.length)
- if nb < s.length then is_writable = false
- end
+ # The string to read from.
+ var source: String
- init(fd: Int)
- do
- is_writable = true
- end
-end
+ # The current position in the string.
+ private var cursor: Int = 0
-class FDIOStream
- super FDIStream
- super FDOStream
- super IOStream
- init(fd: Int)
- do
- self.fd = fd
- is_writable = true
- end
-end
+ redef fun read_char do
+ if cursor < source.length then
+ var c = source[cursor]
-redef interface Object
- # returns first available stream to read or write to
- # return null on interruption (possibly a signal)
- protected fun poll( streams : Sequence[FDStream] ) : nullable FDStream
- do
- var in_fds = new Array[Int]
- var out_fds = new Array[Int]
- var fd_to_stream = new HashMap[Int,FDStream]
- for s in streams do
- var fd = s.fd
- if s isa FDIStream then in_fds.add( fd )
- if s isa FDOStream then out_fds.add( fd )
-
- fd_to_stream[fd] = s
+ cursor += 1
+ return c
+ else
+ return null
end
+ end
- var polled_fd = intern_poll( in_fds, out_fds )
+ redef fun read_byte do
+ if cursor < source.length then
+ var c = source[cursor]
- if polled_fd == null then
- return null
+ cursor += 1
+ return c.ascii
else
- return fd_to_stream[polled_fd]
+ return null
end
end
- private fun intern_poll(in_fds: Array[Int], out_fds: Array[Int]) : nullable Int is extern import Array[Int].length, Array[Int].[], Int.as(nullable Int) `{
- int in_len, out_len, total_len;
- struct pollfd *c_fds;
- sigset_t sigmask;
- int i;
- int first_polled_fd = -1;
- int result;
-
- in_len = Array_of_Int_length( in_fds );
- out_len = Array_of_Int_length( out_fds );
- total_len = in_len + out_len;
- c_fds = malloc( sizeof(struct pollfd) * total_len );
-
- /* input streams */
- for ( i=0; i<in_len; i ++ ) {
- int fd;
- fd = Array_of_Int__index( in_fds, i );
-
- c_fds[i].fd = fd;
- c_fds[i].events = POLLIN;
- }
-
- /* output streams */
- for ( i=0; i<out_len; i ++ ) {
- int fd;
- fd = Array_of_Int__index( out_fds, i );
-
- c_fds[i].fd = fd;
- c_fds[i].events = POLLOUT;
- }
-
- /* poll all fds, unlimited timeout */
- result = poll( c_fds, total_len, -1 );
-
- if ( result > 0 ) {
- /* analyse results */
- for ( i=0; i<total_len; i++ )
- if ( c_fds[i].revents & c_fds[i].events || /* awaited event */
- c_fds[i].revents & POLLHUP ) /* closed */
- {
- first_polled_fd = c_fds[i].fd;
- break;
- }
-
- return Int_as_nullable( first_polled_fd );
- }
- else if ( result < 0 )
- fprintf( stderr, "Error in Stream:poll: %s\n", strerror( errno ) );
-
- return null_Int();
- `}
-end
+ redef fun close do
+ source = ""
+ end
-# Stream to a String. Mainly used for compatibility with OStream type and tests.
-class StringOStream
- super OStream
+ redef fun read_all_bytes do
+ var nslen = source.length - cursor
+ var nns = new NativeString(nslen)
+ source.copy_to_native(nns, nslen, cursor, 0)
+ return new Bytes(nns, nslen, nslen)
+ end
- private var content = new Array[String]
- redef fun to_s do return content.to_s
- redef fun is_writable do return true
- redef fun write(str) do content.add(str)
+ redef fun eof do return cursor >= source.length
end