X-Git-Url: http://nitlanguage.org diff --git a/lib/standard/stream.nit b/lib/standard/stream.nit index e73a3ef..dd1875b 100644 --- a/lib/standard/stream.nit +++ b/lib/standard/stream.nit @@ -11,40 +11,61 @@ # Input and output streams of characters module stream -intrude import ropes +intrude import text::ropes +import error +intrude import bytes in "C" `{ #include - #include - #include #include #include `} -# Abstract stream class -interface IOS +# Any kind of error that could be produced by an operation on Streams +class IOError + super Error +end + +# Any kind of stream to read/write/both to or from a source +abstract class Stream + # Error produced by the file stream + # + # var ifs = new FileReader.open("donotmakethisfile.binx") + # ifs.read_all + # ifs.close + # assert ifs.last_error != null + var last_error: nullable IOError = null + # close the stream fun close is abstract end -# Abstract input streams -interface IStream - super IOS - # Read a character. Return its ASCII value, -1 on EOF or timeout - fun read_char: Int is abstract +# A `Stream` that can be read from +abstract class Reader + super Stream + # Reads a character. Returns `null` on EOF or timeout + fun read_char: nullable Char is abstract + + # Reads a byte. Returns `null` on EOF or timeout + fun read_byte: nullable Byte is abstract + + # Reads a String of at most `i` length + fun read(i: Int): String do return read_bytes(i).to_s # Read at most i bytes - fun read(i: Int): String + fun read_bytes(i: Int): Bytes do - var s = new FlatBuffer.with_capacity(i) + if last_error != null then return new Bytes.empty + var s = new NativeString(i) + var buf = new Bytes(s, 0, 0) while i > 0 and not eof do - var c = read_char - if c >= 0 then - s.add(c.ascii) + var c = read_byte + if c != null then + buf.add c i -= 1 end end - return s.to_s + return buf end # Read a string until the end of the line. @@ -53,7 +74,7 @@ interface IStream # # ~~~ # var txt = "Hello\n\nWorld\n" - # var i = new StringIStream(txt) + # var i = new StringReader(txt) # assert i.read_line == "Hello" # assert i.read_line == "" # assert i.read_line == "World" @@ -66,7 +87,7 @@ interface IStream # # ~~~ # var txt2 = "Hello\r\n\n\rWorld" - # var i2 = new StringIStream(txt2) + # var i2 = new StringReader(txt2) # assert i2.read_line == "Hello" # assert i2.read_line == "" # assert i2.read_line == "\rWorld" @@ -76,6 +97,7 @@ interface IStream # NOTE: Use `append_line_to` if the line terminator needs to be preserved. fun read_line: String do + if last_error != null then return "" if eof then return "" var s = new FlatBuffer append_line_to(s) @@ -88,7 +110,7 @@ interface IStream # # ~~~ # var txt = "Hello\n\nWorld\n" - # var i = new StringIStream(txt) + # var i = new StringReader(txt) # assert i.read_lines == ["Hello", "", "World"] # ~~~ # @@ -105,15 +127,59 @@ interface IStream return res end + # Return an iterator that read each line. + # + # The line terminator '\n' and `\r\n` is removed in each line, + # The line are read with `read_line`. See this method for details. + # + # ~~~ + # var txt = "Hello\n\nWorld\n" + # var i = new StringReader(txt) + # assert i.each_line.to_a == ["Hello", "", "World"] + # ~~~ + # + # Unlike `read_lines` that read all lines at the call, `each_line` is lazy. + # Therefore, the stream should no be closed until the end of the stream. + # + # ~~~ + # i = new StringReader(txt) + # var el = i.each_line + # + # assert el.item == "Hello" + # el.next + # assert el.item == "" + # el.next + # + # i.close + # + # assert not el.is_ok + # # closed before "world" is read + # ~~~ + fun each_line: LineIterator do return new LineIterator(self) + + # Read all the stream until the eof. + # + # The content of the file is returned as a String. + # + # ~~~ + # var txt = "Hello\n\nWorld\n" + # var i = new StringReader(txt) + # assert i.read_all == txt + # ~~~ + fun read_all: String do return read_all_bytes.to_s + # Read all the stream until the eof. - fun read_all: String + # + # The content of the file is returned verbatim. + fun read_all_bytes: Bytes do - var s = new FlatBuffer + if last_error != null then return new Bytes.empty + var s = new Bytes.empty while not eof do - var c = read_char - if c >= 0 then s.add(c.ascii) + var c = read_byte + if c != null then s.add(c) end - return s.to_s + return s end # Read a string until the end of the line and append it to `s`. @@ -124,7 +190,7 @@ interface IStream # # ~~~ # var txt = "Hello\n\nWorld\n" - # var i = new StringIStream(txt) + # var i = new StringReader(txt) # var b = new FlatBuffer # i.append_line_to(b) # assert b == "Hello\n" @@ -139,7 +205,7 @@ interface IStream # a non-eol terminated last line was returned. # # ~~~ - # var i2 = new StringIStream("hello") + # var i2 = new StringReader("hello") # assert not i2.eof # var b2 = new FlatBuffer # i2.append_line_to(b2) @@ -151,14 +217,14 @@ interface IStream # Therefore CARRIAGE RETURN & LINE FEED (`\r\n`) is also recognized. fun append_line_to(s: Buffer) do + if last_error != null then return loop var x = read_char - if x == -1 then + if x == null then if eof then return else - var c = x.ascii - s.chars.push(c) - if c == '\n' then return + s.chars.push(x) + if x == '\n' then return end end end @@ -166,95 +232,270 @@ interface IStream # Is there something to read. # This function returns 'false' if there is something to read. fun eof: Bool is abstract + + # Read the next sequence of non whitespace characters. + # + # Leading whitespace characters are skipped. + # The first whitespace character that follows the result is consumed. + # + # An empty string is returned if the end of the file or an error is encounter. + # + # ~~~ + # var w = new StringReader(" Hello, \n\t World!") + # assert w.read_word == "Hello," + # assert w.read_char == '\n'.ascii + # assert w.read_word == "World!" + # assert w.read_word == "" + # ~~~ + # + # `Char::is_whitespace` determines what is a whitespace. + fun read_word: String + do + var buf = new FlatBuffer + var c = read_nonwhitespace + if c != null then + buf.add(c) + while not eof do + c = read_char + if c == null then break + if c.is_whitespace then break + buf.add(c) + end + end + var res = buf.to_s + return res + end + + # Skip whitespace characters (if any) then return the following non-whitespace character. + # + # Returns the code point of the character. + # Returns `null` on end of file or error. + # + # In fact, this method works like `read_char` except it skips whitespace. + # + # ~~~ + # var w = new StringReader(" \nab\tc") + # assert w.read_nonwhitespace == 'a' + # assert w.read_nonwhitespace == 'b' + # assert w.read_nonwhitespace == 'c' + # assert w.read_nonwhitespace == null + # ~~~ + # + # `Char::is_whitespace` determines what is a whitespace. + fun read_nonwhitespace: nullable Char + do + var c: nullable Char = null + while not eof do + c = read_char + if c == null or not c.is_whitespace then break + end + return c + end end -# IStream capable of declaring if readable without blocking -interface PollableIStream - super IStream +# Iterator returned by `Reader::each_line`. +# See the aforementioned method for details. +class LineIterator + super Iterator[String] + + # The original stream + var stream: Reader + + redef fun is_ok + do + var res = not stream.eof + if not res and close_on_finish then stream.close + return res + end + + redef fun item + do + var line = self.line + if line == null then + line = stream.read_line + end + self.line = line + return line + end + + # The last line read (cache) + private var line: nullable String = null + + redef fun next + do + # force the read + if line == null then item + # drop the line + line = null + end + + # Close the stream when the stream is at the EOF. + # + # Default is false. + var close_on_finish = false is writable + + redef fun finish + do + if close_on_finish then stream.close + end +end + +# `Reader` capable of declaring if readable without blocking +abstract class PollableReader + super Reader # Is there something to read? (without blocking) fun poll_in: Bool is abstract end -# Abstract output stream -interface OStream - super IOS +# A `Stream` that can be written to +abstract class Writer + super Stream + + # Writes bytes from `s` + fun write_bytes(s: Bytes) is abstract + # write a string fun write(s: Text) is abstract + # Write a single byte + fun write_byte(value: Byte) is abstract + # Can the stream be used to write fun is_writable: Bool is abstract end -# Things that can be efficienlty writen to a OStream +# Things that can be efficienlty written to a `Writer` # -# The point of this interface it to allow is instance to be efficenty -# writen into a OStream without having to allocate a big String object +# The point of this interface is to allow the instance to be efficiently +# written into a `Writer`. # -# ready-to-save documents usually provide this interface. -interface Streamable +# Ready-to-save documents usually provide this interface. +interface Writable # Write itself to a `stream` # The specific logic it let to the concrete subclasses - fun write_to(stream: OStream) is abstract + fun write_to(stream: Writer) is abstract # Like `write_to` but return a new String (may be quite large) # - # This funtionnality is anectodical, since the point + # This funtionality is anectodical, since the point # of streamable object to to be efficienlty written to a # stream without having to allocate and concatenate strings fun write_to_string: String do - var stream = new StringOStream + var stream = new StringWriter write_to(stream) return stream.to_s end end redef class Text - super Streamable + super Writable redef fun write_to(stream) do stream.write(self) end -# Input streams with a buffer -abstract class BufferedIStream - super IStream +# Input streams with a buffered input for efficiency purposes +abstract class BufferedReader + super Reader redef fun read_char do - if _buffer_pos >= _buffer.length then - fill_buffer - end - if _buffer_pos >= _buffer.length then - return -1 + if last_error != null then return null + if eof then + last_error = new IOError("Stream has reached eof") + return null end - var c = _buffer.chars[_buffer_pos] + # TODO: Fix when supporting UTF-8 + var c = _buffer[_buffer_pos].to_i.ascii _buffer_pos += 1 - return c.ascii + return c end - redef fun read(i) + redef fun read_byte do - if _buffer.length == _buffer_pos then - if not eof then - return read(i) - end - return "" + if last_error != null then return null + if eof then + last_error = new IOError("Stream has reached eof") + return null + end + var c = _buffer[_buffer_pos] + _buffer_pos += 1 + return c + end + + fun buffer_reset do + _buffer_length = 0 + _buffer_pos = 0 + end + + # Peeks up to `n` bytes in the buffer + # + # The operation does not consume the buffer + # + # ~~~nitish + # var x = new FileReader.open("File.txt") + # assert x.peek(5) == x.read(5) + # ~~~ + fun peek(i: Int): Bytes do + if eof then return new Bytes.empty + var remsp = _buffer_length - _buffer_pos + if i <= remsp then + var bf = new Bytes.with_capacity(i) + bf.append_ns_from(_buffer, i, _buffer_pos) + return bf end - if _buffer_pos + i >= _buffer.length then - var from = _buffer_pos - _buffer_pos = _buffer.length - return _buffer.substring_from(from).to_s + var bf = new Bytes.with_capacity(i) + bf.append_ns_from(_buffer, remsp, _buffer_pos) + _buffer_pos = _buffer_length + read_intern(i - bf.length, bf) + remsp = _buffer_length - _buffer_pos + var full_len = bf.length + remsp + if full_len > _buffer_capacity then + var c = _buffer_capacity + while c < full_len do c = c * 2 + 2 + _buffer_capacity = c end - _buffer_pos += i - return _buffer.substring(_buffer_pos - i, i).to_s + var nns = new NativeString(_buffer_capacity) + bf.items.copy_to(nns, bf.length, 0, 0) + _buffer.copy_to(nns, remsp, _buffer_pos, bf.length) + _buffer = nns + _buffer_pos = 0 + _buffer_length = full_len + return bf end - redef fun read_all + redef fun read_bytes(i) do - var s = new FlatBuffer + if last_error != null then return new Bytes.empty + var buf = new Bytes.with_capacity(i) + read_intern(i, buf) + return buf + end + + # Fills `buf` with at most `i` bytes read from `self` + private fun read_intern(i: Int, buf: Bytes): Int do + if eof then return 0 + var p = _buffer_pos + var bufsp = _buffer_length - p + if bufsp >= i then + _buffer_pos += i + buf.append_ns_from(_buffer, i, p) + return i + end + _buffer_pos = _buffer_length + var readln = _buffer_length - p + buf.append_ns_from(_buffer, readln, p) + var rd = read_intern(i - readln, buf) + return rd + readln + end + + redef fun read_all_bytes + do + if last_error != null then return new Bytes.empty + var s = new Bytes.with_capacity(10) while not eof do var j = _buffer_pos - var k = _buffer.length + var k = _buffer_length while j < k do s.add(_buffer[j]) j += 1 @@ -262,7 +503,7 @@ abstract class BufferedIStream _buffer_pos = j fill_buffer end - return s.to_s + return s end redef fun append_line_to(s) @@ -270,11 +511,13 @@ abstract class BufferedIStream loop # First phase: look for a '\n' var i = _buffer_pos - while i < _buffer.length and _buffer.chars[i] != '\n' do i += 1 + while i < _buffer_length and _buffer[i] != 0xAu8 do + i += 1 + end var eol - if i < _buffer.length then - assert _buffer.chars[i] == '\n' + if i < _buffer_length then + assert _buffer[i] == 0xAu8 i += 1 eol = true else @@ -289,7 +532,7 @@ abstract class BufferedIStream # Copy from the buffer to the string var j = _buffer_pos while j < i do - s.add(_buffer.chars[j]) + s.bytes.add(_buffer[j]) j += 1 end _buffer_pos = i @@ -311,176 +554,60 @@ abstract class BufferedIStream redef fun eof do - if _buffer_pos < _buffer.length then return false + if _buffer_pos < _buffer_length then return false if end_reached then return true fill_buffer - return _buffer_pos >= _buffer.length and end_reached + return _buffer_pos >= _buffer_length and end_reached end # The buffer - private var buffer: nullable FlatBuffer = null + private var buffer: NativeString = new NativeString(0) # The current position in the buffer - private var buffer_pos: Int = 0 + private var buffer_pos = 0 + + # Length of the current buffer (i.e. nuber of bytes in the buffer) + private var buffer_length = 0 + + # Capacity of the buffer + private var buffer_capacity = 0 # Fill the buffer protected fun fill_buffer is abstract - # Is the last fill_buffer reach the end + # Has the last fill_buffer reached the end protected fun end_reached: Bool is abstract # Allocate a `_buffer` for a given `capacity`. protected fun prepare_buffer(capacity: Int) do - _buffer = new FlatBuffer.with_capacity(capacity) + _buffer = new NativeString(capacity) _buffer_pos = 0 # need to read + _buffer_length = 0 + _buffer_capacity = capacity end end -# An Input/Output Stream -interface IOStream - super IStream - super OStream -end - -##############################################################" - -# A File Descriptor Stream. -abstract class FDStream - super IOS - # File description - var fd: Int - - redef fun close do native_close(fd) - - private fun native_close(i: Int): Int is extern "stream_FDStream_FDStream_native_close_1" - private fun native_read_char(i: Int): Int is extern "stream_FDStream_FDStream_native_read_char_1" - private fun native_read(i: Int, buf: NativeString, len: Int): Int is extern "stream_FDStream_FDStream_native_read_3" - private fun native_write(i: Int, buf: NativeString, len: Int): Int is extern "stream_FDStream_FDStream_native_write_3" - private fun native_write_char(i: Int, c: Char): Int is extern "stream_FDStream_FDStream_native_write_char_2" -end - -# An Input File Descriptor Stream. -class FDIStream - super FDStream - super IStream - redef var eof: Bool = false - - redef fun read_char - do - var nb = native_read_char(fd) - if nb == -1 then eof = true - return nb - end -end - -# An Output File Descriptor Stream. -class FDOStream - super FDStream - super OStream - redef var is_writable = true - - redef fun write(s) - do - var nb = native_write(fd, s.to_cstring, s.length) - if nb < s.length then is_writable = false - end -end - -# An Input/Output File Descriptor Stream. -class FDIOStream - super FDIStream - super FDOStream - super IOStream +# A `Stream` that can be written to and read from +abstract class Duplex + super Reader + super Writer end -redef interface Object - # returns first available stream to read or write to - # return null on interruption (possibly a signal) - protected fun poll( streams : Sequence[FDStream] ) : nullable FDStream - do - var in_fds = new Array[Int] - var out_fds = new Array[Int] - var fd_to_stream = new HashMap[Int,FDStream] - for s in streams do - var fd = s.fd - if s isa FDIStream then in_fds.add( fd ) - if s isa FDOStream then out_fds.add( fd ) - - fd_to_stream[fd] = s - end - - var polled_fd = intern_poll( in_fds, out_fds ) - - if polled_fd == null then - return null - else - return fd_to_stream[polled_fd] - end - end - - private fun intern_poll(in_fds: Array[Int], out_fds: Array[Int]) : nullable Int is extern import Array[Int].length, Array[Int].[], Int.as(nullable Int) `{ - int in_len, out_len, total_len; - struct pollfd *c_fds; - sigset_t sigmask; - int i; - int first_polled_fd = -1; - int result; - - in_len = Array_of_Int_length( in_fds ); - out_len = Array_of_Int_length( out_fds ); - total_len = in_len + out_len; - c_fds = malloc( sizeof(struct pollfd) * total_len ); - - /* input streams */ - for ( i=0; i 0 ) { - /* analyse results */ - for ( i=0; i= source.length end