From: Jean Privat Date: Tue, 26 May 2015 10:09:33 +0000 (-0400) Subject: Merge: File and Bytes X-Git-Tag: v0.7.5~32 X-Git-Url: http://nitlanguage.org?hp=52ecf977412e09bbee42bcf7b4268c393e9a8a3e Merge: File and Bytes A bit of a fix in the context of issues #1267 and #1262. The bytes module has a `ByteBuffer` class, while it is still working with `Int` at the moment, when Nit correctly supports `Bytes`, we can change it to a real `ByteBuffer` This class is highly recommended when working with byte streams and collections. As such, file operations are related to this module since most of the operations are working on bytes, what they mean is up to the end-user to figure. Pull-Request: #1309 Reviewed-by: Alexandre Terrasa Reviewed-by: Jean Privat Reviewed-by: Romain Chanoir Reviewed-by: Ait younes Mehdi Adel Reviewed-by: Alexis Laferrière --- diff --git a/contrib/online_ide/sources/nit/pnacl_nit.nit b/contrib/online_ide/sources/nit/pnacl_nit.nit index 74c7be0..2017a0b 100644 --- a/contrib/online_ide/sources/nit/pnacl_nit.nit +++ b/contrib/online_ide/sources/nit/pnacl_nit.nit @@ -72,7 +72,7 @@ redef class FileReader self.path = path var file = sys.files[path] prepare_buffer(file.length) - _buffer.append(file) + path.copy_to_native(_buffer, file.length, 0, 0) end redef fun close @@ -82,7 +82,7 @@ redef class FileReader redef fun fill_buffer do - _buffer.clear + buffer_reset end_reached = true end diff --git a/lib/bitmap/bitmap.nit b/lib/bitmap/bitmap.nit index 0343b8e..9be1bc2 100644 --- a/lib/bitmap/bitmap.nit +++ b/lib/bitmap/bitmap.nit @@ -123,14 +123,20 @@ class Bitmap # =============== Bitmap header ================ for x in [0..13] do - bitmap_header[x] = fileReader.read(1)[0].ascii + var b = fileReader.read_byte + if b == null then + return + end + bitmap_header[x] = b end self.file_size = get_value(bitmap_header.subarray(2, 4)) self.data_offset = get_value(bitmap_header.subarray(10, 4)) # =============== DIB header ================ for x in [0..39] do - dib_header[x] = fileReader.read(1)[0].ascii + var b = fileReader.read_byte + if b == null then return + dib_header[x] = b end var dib_size = get_value(dib_header.subarray(0, 4)) # only support BITMAPINFOHEADER @@ -159,9 +165,11 @@ class Bitmap var row = new Array[Int].with_capacity(self.width) for y in [0..self.width[ do - var red = fileReader.read(1)[0].ascii * 256 * 256 - var green = fileReader.read(1)[0].ascii * 256 - var blue = fileReader.read(1)[0].ascii + var bts = fileReader.read_bytes(3) + if bts.length != 3 then return + var red = bts[0] << 16 + var green = bts[1] << 8 + var blue = bts[2] row.add(red + green + blue) end self.data.add(row) @@ -170,18 +178,6 @@ class Bitmap fileReader.close end #end of load_from_file method - # Reads in a series of bytes from the FileReader when loading a Bitmap from a file - # FileReader.read(1) is used due to https://github.com/privat/nit/issues/1264 - private fun read_chars(fr: FileReader, howMany: Int): String - do - var s = "" - for x in [1..howMany] - do - s += fr.read(1) - end - return s - end - # Converts the value contained in two or four bytes into an Int. Since Nit # does not have a byte type, Int is used private fun get_value(array: Array[Int]): Int diff --git a/lib/pnacl.nit b/lib/pnacl.nit index 516c860..248a4d4 100644 --- a/lib/pnacl.nit +++ b/lib/pnacl.nit @@ -653,9 +653,11 @@ class PnaclStream # fill_buffer now checks for a message in the message queue which is filled by user inputs. redef fun fill_buffer do - _buffer.clear _buffer_pos = 0 - _buffer.append check_message.to_s + var nns = check_message + var nslen = nns.cstring_length + _buffer_length = nslen + nns.copy_to(buffer, nslen, 0, 0) end end diff --git a/lib/socket/socket.nit b/lib/socket/socket.nit index b3cb2f9..d553540 100644 --- a/lib/socket/socket.nit +++ b/lib/socket/socket.nit @@ -57,7 +57,7 @@ class TCPStream # Creates a socket connection to host `host` on port `port` init connect(host: String, port: Int) do - _buffer = new FlatBuffer + _buffer = new NativeString(1024) _buffer_pos = 0 socket = new NativeSocket.socket(new NativeSocketAddressFamilies.af_inet, new NativeSocketTypes.sock_stream, new NativeSocketProtocolFamilies.pf_null) @@ -84,7 +84,7 @@ class TCPStream # Creates a client socket, this is meant to be used by accept only private init server_side(h: SocketAcceptResult) do - _buffer = new FlatBuffer + _buffer = new NativeString(1024) _buffer_pos = 0 socket = h.socket addrin = h.addr_in @@ -110,7 +110,7 @@ class TCPStream # timeout : Time in milliseconds before stopping to wait for events fun ready_to_read(timeout: Int): Bool do - if _buffer_pos < _buffer.length then return true + if _buffer_pos < _buffer_length then return true if end_reached then return false var events = [new NativeSocketPollValues.pollin] return pollin(events, timeout).length != 0 @@ -153,6 +153,11 @@ class TCPStream socket.write_byte value end + redef fun write_bytes(s) do + if closed then return + socket.write(s.to_s) + end + fun write_ln(msg: Text) do if end_reached then return @@ -162,7 +167,7 @@ class TCPStream redef fun fill_buffer do - _buffer.clear + _buffer_length = 0 _buffer_pos = 0 if not connected then return var read = socket.read @@ -170,7 +175,17 @@ class TCPStream close end_reached = true end - _buffer.append(read) + enlarge(_buffer_capacity + read.length) + read.copy_to_native(_buffer, read.length, 0, 0) + _buffer_length = read.length + end + + fun enlarge(len: Int) do + if _buffer_capacity >= len then return + while _buffer_capacity < len do _buffer_capacity *= 2 + var ns = new NativeString(_buffer_capacity) + _buffer.copy_to(ns, _buffer_length - _buffer_pos, _buffer_pos, 0) + _buffer = ns end redef fun close diff --git a/lib/socket/socket_c.nit b/lib/socket/socket_c.nit index a574485..627fb06 100644 --- a/lib/socket/socket_c.nit +++ b/lib/socket/socket_c.nit @@ -137,16 +137,14 @@ extern class NativeSocket `{ int* `} return write(*recv, &value, 1); `} - fun read: String import NativeString.to_s_with_length `{ + fun read: String import NativeString.to_s_with_length, NativeString.to_s_with_copy `{ static char c[1024]; - int n = read(*recv, c, 1024); + int n = read(*recv, c, 1023); if(n < 0) { return NativeString_to_s_with_length("",0); } - char* ret = malloc(n + 1); - memcpy(ret, c, n); - ret[n] = '\0'; - return NativeString_to_s_with_length(ret, n); + c[n] = 0; + return NativeString_to_s_with_copy(c); `} # Sets an option for the socket diff --git a/lib/standard/file.nit b/lib/standard/file.nit index 1490ee7..3548a25 100644 --- a/lib/standard/file.nit +++ b/lib/standard/file.nit @@ -113,25 +113,24 @@ class FileReader return end end_reached = false - _buffer_pos = 0 - _buffer.clear + buffer_reset end redef fun close do super - _buffer.clear + buffer_reset end_reached = true end redef fun fill_buffer do - var nb = _file.io_read(_buffer.items, _buffer.capacity) + var nb = _file.io_read(_buffer, _buffer_capacity) if nb <= 0 then end_reached = true nb = 0 end - _buffer.length = nb + _buffer_length = nb _buffer_pos = 0 end @@ -179,18 +178,23 @@ class FileWriter super FileStream super Writer + redef fun write_bytes(s) do + if last_error != null then return + if not _is_writable then + last_error = new IOError("cannot write to non-writable stream") + return + end + write_native(s.items, s.length) + end + redef fun write(s) do if last_error != null then return if not _is_writable then - last_error = new IOError("Cannot write to non-writable stream") + last_error = new IOError("cannot write to non-writable stream") return end - if s isa FlatText then - write_native(s.to_cstring, s.length) - else - for i in s.substrings do write_native(i.to_cstring, i.length) - end + for i in s.substrings do write_native(i.to_cstring, i.length) end redef fun write_byte(value) @@ -435,10 +439,12 @@ class Path # ~~~ # # See `Reader::read_all` for details. - fun read_all: String + fun read_all: String do return read_all_bytes.to_s + + fun read_all_bytes: Bytes do var s = open_ro - var res = s.read_all + var res = s.read_all_bytes s.close return res end diff --git a/lib/standard/stream.nit b/lib/standard/stream.nit index 7c4e606..b901596 100644 --- a/lib/standard/stream.nit +++ b/lib/standard/stream.nit @@ -13,6 +13,7 @@ module stream intrude import ropes import error +intrude import bytes in "C" `{ #include @@ -48,19 +49,23 @@ abstract class Reader # Reads a byte. Returns `null` on EOF or timeout fun read_byte: nullable Int is abstract + # Reads a String of at most `i` length + fun read(i: Int): String do return read_bytes(i).to_s + # Read at most i bytes - fun read(i: Int): String + fun read_bytes(i: Int): Bytes do - if last_error != null then return "" - var s = new FlatBuffer.with_capacity(i) + if last_error != null then return new Bytes.empty + var s = new NativeString(i) + var buf = new Bytes(s, 0, 0) while i > 0 and not eof do - var c = read_char + var c = read_byte if c != null then - s.add(c) + buf.add c i -= 1 end end - return s.to_s + return buf end # Read a string until the end of the line. @@ -154,22 +159,27 @@ abstract class Reader # Read all the stream until the eof. # - # The content of the file is returned verbatim. + # The content of the file is returned as a String. # # ~~~ # var txt = "Hello\n\nWorld\n" # var i = new StringReader(txt) # assert i.read_all == txt # ~~~ - fun read_all: String + fun read_all: String do return read_all_bytes.to_s + + # Read all the stream until the eof. + # + # The content of the file is returned verbatim. + fun read_all_bytes: Bytes do - if last_error != null then return "" - var s = new FlatBuffer + if last_error != null then return new Bytes.empty + var s = new Bytes.empty while not eof do - var c = read_char + var c = read_byte if c != null then s.add(c) end - return s.to_s + return s end # Read a string until the end of the line and append it to `s`. @@ -342,6 +352,10 @@ end # A `Stream` that can be written to abstract class Writer super Stream + + # Writes bytes from `s` + fun write_bytes(s: Bytes) is abstract + # write a string fun write(s: Text) is abstract @@ -365,7 +379,7 @@ interface Writable # Like `write_to` but return a new String (may be quite large) # - # This funtionnality is anectodical, since the point + # This funtionality is anectodical, since the point # of streamable object to to be efficienlty written to a # stream without having to allocate and concatenate strings fun write_to_string: String @@ -408,66 +422,87 @@ abstract class BufferedReader return c end - # Peeks up to `n` bytes in the buffer, returns an empty string on EOF + fun buffer_reset do + _buffer_length = 0 + _buffer_pos = 0 + end + + # Peeks up to `n` bytes in the buffer # # The operation does not consume the buffer # # ~~~nitish - # var x = new FileReader("File.txt") - # assert x.peek(5) == x.read(5) + # var x = new FileReader.open("File.txt") + # assert x.peek(5) == x.read(5) # ~~~ - fun peek(i: Int): String do - if eof then return "" - var b = new FlatBuffer.with_capacity(i) - while i > 0 and not eof do - b.add _buffer[_buffer_pos] - _buffer_pos += 1 - i -= 1 + fun peek(i: Int): Bytes do + if eof then return new Bytes.empty + var remsp = _buffer_length - _buffer_pos + if i <= remsp then + var bf = new Bytes.with_capacity(i) + bf.append_ns_from(_buffer, i, _buffer_pos) + return bf end - var nbuflen = b.length + (_buffer.length - _buffer_pos) - var nbuf = new FlatBuffer.with_capacity(nbuflen) - nbuf.append(b) - while _buffer_pos < _buffer.length do - nbuf.add(_buffer[_buffer_pos]) - _buffer_pos += 1 + var bf = new Bytes.with_capacity(i) + bf.append_ns_from(_buffer, remsp, _buffer_pos) + _buffer_pos = _buffer_length + read_intern(i - bf.length, bf) + remsp = _buffer_length - _buffer_pos + var full_len = bf.length + remsp + if full_len > _buffer_capacity then + var c = _buffer_capacity + while c < full_len do c = c * 2 + 2 + _buffer_capacity = c end + var nns = new NativeString(_buffer_capacity) + bf.items.copy_to(nns, bf.length, 0, 0) + _buffer.copy_to(nns, remsp, _buffer_pos, bf.length) + _buffer = nns _buffer_pos = 0 - _buffer = nbuf - return b.to_s + _buffer_length = full_len + return bf end - redef fun read(i) + redef fun read_bytes(i) do - if last_error != null then return "" - if eof then return "" + if last_error != null then return new Bytes.empty + var buf = new Bytes.with_capacity(i) + read_intern(i, buf) + return buf + end + + # Fills `buf` with at most `i` bytes read from `self` + private fun read_intern(i: Int, buf: Bytes): Int do + if eof then return 0 var p = _buffer_pos - var bufsp = _buffer.length - p + var bufsp = _buffer_length - p if bufsp >= i then _buffer_pos += i - return _buffer.substring(p, i).to_s + buf.append_ns_from(_buffer, i, p) + return i end - _buffer_pos = _buffer.length - var readln = _buffer.length - p - var s = _buffer.substring(p, readln).to_s - fill_buffer - return s + read(i - readln) + _buffer_pos = _buffer_length + var readln = _buffer_length - p + buf.append_ns_from(_buffer, readln, p) + var rd = read_intern(i - readln, buf) + return rd + readln end - redef fun read_all + redef fun read_all_bytes do - if last_error != null then return "" - var s = new FlatBuffer + if last_error != null then return new Bytes.empty + var s = new Bytes.with_capacity(10) while not eof do var j = _buffer_pos - var k = _buffer.length + var k = _buffer_length while j < k do - s.add(_buffer[j]) + s.add(_buffer[j].ascii) j += 1 end _buffer_pos = j fill_buffer end - return s.to_s + return s end redef fun append_line_to(s) @@ -475,10 +510,12 @@ abstract class BufferedReader loop # First phase: look for a '\n' var i = _buffer_pos - while i < _buffer.length and _buffer[i] != '\n' do i += 1 + while i < _buffer_length and _buffer[i] != '\n' do + i += 1 + end var eol - if i < _buffer.length then + if i < _buffer_length then assert _buffer[i] == '\n' i += 1 eol = true @@ -516,29 +553,37 @@ abstract class BufferedReader redef fun eof do - if _buffer_pos < _buffer.length then return false + if _buffer_pos < _buffer_length then return false if end_reached then return true fill_buffer - return _buffer_pos >= _buffer.length and end_reached + return _buffer_pos >= _buffer_length and end_reached end # The buffer - private var buffer: nullable FlatBuffer = null + private var buffer: NativeString = new NativeString(0) # The current position in the buffer - private var buffer_pos: Int = 0 + private var buffer_pos = 0 + + # Length of the current buffer (i.e. nuber of bytes in the buffer) + private var buffer_length = 0 + + # Capacity of the buffer + private var buffer_capacity = 0 # Fill the buffer protected fun fill_buffer is abstract - # Is the last fill_buffer reach the end + # Has the last fill_buffer reached the end protected fun end_reached: Bool is abstract # Allocate a `_buffer` for a given `capacity`. protected fun prepare_buffer(capacity: Int) do - _buffer = new FlatBuffer.with_capacity(capacity) + _buffer = new NativeString(capacity) _buffer_pos = 0 # need to read + _buffer_length = 0 + _buffer_capacity = capacity end end @@ -557,6 +602,11 @@ class StringWriter private var content = new Array[String] redef fun to_s do return content.to_s redef fun is_writable do return not closed + + redef fun write_bytes(b) do + content.add(b.to_s) + end + redef fun write(str) do assert not closed @@ -607,11 +657,11 @@ class StringReader source = "" end - redef fun read_all do - var c = cursor - cursor = source.length - if c == 0 then return source - return source.substring_from(c) + redef fun read_all_bytes do + var nslen = source.length - cursor + var nns = new NativeString(nslen) + source.copy_to_native(nns, nslen, cursor, 0) + return new Bytes(nns, nslen, nslen) end redef fun eof do return cursor >= source.length diff --git a/lib/websocket/websocket.nit b/lib/websocket/websocket.nit index a22c5b2..6eac618 100644 --- a/lib/websocket/websocket.nit +++ b/lib/websocket/websocket.nit @@ -23,6 +23,7 @@ import sha1 import base64 intrude import standard::stream +intrude import standard::bytes # Websocket compatible listener # @@ -65,8 +66,10 @@ class WebsocketConnection super TCPStream init do - _buffer = new FlatBuffer + _buffer = new NativeString(1024) _buffer_pos = 0 + _buffer_capacity = 1024 + _buffer_length = 0 var headers = parse_handshake var resp = handshake_response(headers) @@ -119,22 +122,28 @@ class WebsocketConnection end # Frames a text message to be sent to a client - private fun frame_message(msg: String): String + private fun frame_message(msg: String): Bytes do - var ans_buffer = new FlatBuffer + var ans_buffer = new Bytes.with_capacity(msg.length) # Flag for final frame set to 1 # opcode set to 1 (for text) - ans_buffer.add(129.ascii) + ans_buffer.add(129) if msg.length < 126 then - ans_buffer.add(msg.length.ascii) + ans_buffer.add(msg.length) end if msg.length >= 126 and msg.length <= 65535 then - ans_buffer.add(126.ascii) - ans_buffer.add(msg.length.rshift(8).ascii) - ans_buffer.add(msg.length.ascii) + ans_buffer.add(126) + ans_buffer.add(msg.length.rshift(8)) + ans_buffer.add(msg.length) end - ans_buffer.append(msg) - return ans_buffer.to_s + if msg isa FlatString then + ans_buffer.append_ns_from(msg.items, msg.length, msg.index_from) + else + for i in msg.substrings do + ans_buffer.append_ns_from(i.as(FlatString).items, i.length, i.as(FlatString).index_from) + end + end + return ans_buffer end # Reads an HTTP frame @@ -149,6 +158,7 @@ class WebsocketConnection # Gets the message from the client, unpads it and reconstitutes the message private fun unpad_message do var fin = false + var bf = new Bytes.empty while not fin do var fst_byte = client.read_byte var snd_byte = client.read_byte @@ -174,10 +184,10 @@ class WebsocketConnection if fin_flag != 0 then fin = true var opcode = fst_byte.bin_and(15) if opcode == 9 then - _buffer.add(138.ascii) - _buffer.add('\0') - client.write(_buffer.to_s) - _buffer_pos += 2 + bf.add(138) + bf.add(0) + client.write(bf.to_s) + _buffer_pos = _buffer_length return end if opcode == 8 then @@ -192,76 +202,68 @@ class WebsocketConnection var len = snd_byte.bin_and(127) var payload_ext_len = 0 if len == 126 then - var tmp = client.read(2) + var tmp = client.read_bytes(2) if tmp.length != 2 then last_error = new IOError("Error: received interrupted frame") client.close return end - payload_ext_len = tmp[1].ascii + tmp[0].ascii.lshift(8) + payload_ext_len = tmp[1] + tmp[0].lshift(8) else if len == 127 then # 64 bits for length are not supported, # only the last 32 will be interpreted as a Nit Integer - var tmp = client.read(8) + var tmp = client.read_bytes(8) if tmp.length != 8 then last_error = new IOError("Error: received interrupted frame") client.close return end for pos in [0 .. tmp.length[ do - var i = tmp[pos].ascii + var i = tmp[pos] payload_ext_len += i.lshift(8 * (7 - pos)) end end if mask_flag != 0 then + var mask = client.read_bytes(4).items if payload_ext_len != 0 then - var msg = client.read(payload_ext_len+4) - var mask = msg.substring(0,4) - _buffer.append(unmask_message(mask, msg.substring(4, payload_ext_len))) - else - if len == 0 then - return - end - var msg = client.read(len+4) - var mask = msg.substring(0,4) - _buffer.append(unmask_message(mask, msg.substring(4, len))) + len = payload_ext_len end + var msg = client.read_bytes(len).items + bf.append_ns(unmask_message(mask, msg, len), len) end end + _buffer = bf.items + _buffer_length = bf.length end # Unmasks a message sent by a client - private fun unmask_message(key: String, message: String): String + private fun unmask_message(key: NativeString, message: NativeString, len: Int): NativeString do - var return_message = new FlatBuffer.with_capacity(message.length) - var msg_iter = message.chars.iterator + var return_message = new NativeString(len) - while msg_iter.is_ok do - return_message.chars[msg_iter.index] = msg_iter.item.ascii.bin_xor(key.chars[msg_iter.index%4].ascii).ascii - msg_iter.next + for i in [0 .. len[ do + return_message[i] = message[i].ascii.bin_xor(key[i%4].ascii).ascii end - return return_message.to_s + return return_message end # Checks if a connection to a client is available redef fun connected do return client.connected - redef fun write(msg) - do - client.write(frame_message(msg.to_s)) - end + redef fun write_bytes(s) do client.write_bytes(frame_message(s.to_s)) + + redef fun write(msg) do client.write(frame_message(msg.to_s).to_s) redef fun is_writable do return client.connected redef fun fill_buffer do - _buffer.clear - _buffer_pos = 0 + buffer_reset unpad_message end - redef fun end_reached do return client._buffer_pos >= client._buffer.length and client.end_reached + redef fun end_reached do return client._buffer_pos >= client._buffer_length and client.end_reached # Is there some data available to be read ? fun can_read(timeout: Int): Bool do return client.ready_to_read(timeout)