lib/standard: Update libs for the support of UTF-8
authorLucas Bajolet <r4pass@hotmail.com>
Tue, 21 Jul 2015 17:46:08 +0000 (13:46 -0400)
committerLucas Bajolet <r4pass@hotmail.com>
Thu, 23 Jul 2015 15:17:32 +0000 (11:17 -0400)
Signed-off-by: Lucas Bajolet <r4pass@hotmail.com>

contrib/pep8analysis/src/parser/lexer.nit
lib/buffered_ropes.nit
lib/ropes_debug.nit
lib/standard/file.nit
lib/standard/stream.nit
lib/standard/text/abstract_text.nit
lib/standard/text/flat.nit
lib/standard/text/ropes.nit
lib/websocket/websocket.nit

index 96f5430..912336c 100644 (file)
@@ -338,6 +338,7 @@ class Lexer
                                dfa_state = -1
                        else
                                var c = string[sp].ascii
+                               if c >= 255 then c = 255
                                sp += 1
 
                                var cr = _cr
index a2a8023..1f92337 100644 (file)
@@ -94,6 +94,9 @@ private class Leaf
        var bns: NativeString is noinit
        redef var length is noinit
 
+       # Unsafe, but since it is an experiment, don't mind
+       redef fun bytelen do return length
+
        redef fun empty do return new Leaf(new ManualBuffer)
 
        redef fun to_cstring do
@@ -163,14 +166,14 @@ private class Leaf
                        var bpos = buf.pos
                        var sits = s.items
                        if bpos == mlen then
-                               sits.copy_to(buf.ns, slen, s.index_from, bpos)
+                               sits.copy_to(buf.ns, slen, s.first_byte, bpos)
                                buf.pos = bpos + slen
                                return new Leaf(buf)
                        else
                                var b = new ManualBuffer
                                var nbns = b.ns
                                bns.copy_to(nbns, mlen, 0, 0)
-                               sits.copy_to(nbns, slen, s.index_from, mlen)
+                               sits.copy_to(nbns, slen, s.first_byte, mlen)
                                b.pos = nlen
                                return new Leaf(b)
                        end
@@ -219,7 +222,7 @@ redef class Concat
                for i in substrings do
                        var ilen = i.length
                        if i isa FlatString then
-                               i.items.copy_to(ns, ilen, i.index_from, off)
+                               i.items.copy_to(ns, ilen, i.first_byte, off)
                        else if i isa Leaf then
                                i.buf.ns.copy_to(ns, ilen, 0, off)
                        else
@@ -261,8 +264,8 @@ redef class FlatString
                if s isa FlatString then
                        if slen + mlen > maxlen then return new Concat(self, s)
                        var mits = items
-                       var sifrom = s.index_from
-                       var mifrom = index_from
+                       var sifrom = s.first_byte
+                       var mifrom = first_byte
                        var sits = s.items
                        var b = new ManualBuffer
                        var bns = b.ns
@@ -277,7 +280,7 @@ redef class FlatString
                        return new Concat(sl + self, s.right)
                else if s isa Leaf then
                        if slen + mlen > maxlen then return new Concat(self, s)
-                       var mifrom = index_from
+                       var mifrom = first_byte
                        var sb = s.buf
                        var b = new ManualBuffer
                        var bns = b.ns
@@ -323,13 +326,13 @@ redef class Array[E]
                        var tmp = na[i]
                        var tpl = tmp.length
                        if tmp isa FlatString then
-                               tmp.items.copy_to(ns, tpl, tmp.index_from, off)
+                               tmp.items.copy_to(ns, tpl, tmp.first_byte, off)
                                off += tpl
                        else
                                for j in tmp.substrings do
                                        var slen = j.length
                                        if j isa FlatString then
-                                               j.items.copy_to(ns, slen, j.index_from, off)
+                                               j.items.copy_to(ns, slen, j.first_byte, off)
                                        else if j isa Leaf then
                                                j.buf.ns.copy_to(ns, slen, 0, off)
                                        end
index a87e8d4..8482352 100644 (file)
@@ -45,7 +45,7 @@ redef class RopeBuffer
                s += "n{object_id} -> n{str.object_id} [label = \"str\"];\n"
                s += str.internal_to_dot
                s += "n{object_id} -> n{ns.object_id} [label = \"ns\"];\n"
-               s += "n{ns.object_id}[label = \"NativeString\", content=\"{ns.to_s_with_length(rpos)}\"];\n"
+               s += "n{ns.object_id}[label = \"Items\", content=\"{ns}\"];\n"
                return s
        end
 end
@@ -53,14 +53,14 @@ end
 redef class FlatString
        redef fun internal_to_dot: String
        do
-               return "n{object_id} [label=\"FlatString\\nindex_from = {index_from}\\nindex_to = {index_to}\\nNativeString = {items.to_s_with_length(items.cstring_length)}\"];\n"
+               return "n{object_id} [label=\"FlatString\\nlength = {length}\\nbytelen = {bytelen}\\nfirst_byte = {first_byte}\\nlast_byte = {last_byte}\\nText = {self.escape_to_dot}\"];\n"
        end
 end
 
 redef class FlatBuffer
        redef fun internal_to_dot: String
        do
-               return "n{object_id} [label=\"FlatBuffer\\length = {length}\\ncapacity = {capacity}\\nitems = {items.to_s_with_length(items.cstring_length)}\"];\n"
+               return "n{object_id} [label=\"FlatBuffer\\nbytelen = {bytelen}\\nlength = {length}\\ncapacity = {capacity}\\nText = {escape_to_dot}\"];\n"
        end
 end
 
index 4fdafb1..f28f522 100644 (file)
@@ -679,7 +679,7 @@ redef class Text
 
        private fun write_native_to(s: FileWriter)
        do
-               for i in substrings do s.write_native(i.to_cstring, 0, i.length)
+               for i in substrings do s.write_native(i.to_cstring, 0, i.bytelen)
        end
 end
 
@@ -1099,7 +1099,7 @@ end
 redef class FlatString
        redef fun write_native_to(s)
        do
-               s.write_native(items, index_from, length)
+               s.write_native(items, first_byte, bytelen)
        end
 end
 
index dd1875b..7218514 100644 (file)
@@ -527,7 +527,7 @@ abstract class BufferedReader
                        # if there is something to append
                        if i > _buffer_pos then
                                # Enlarge the string (if needed)
-                               s.enlarge(s.length + i - _buffer_pos)
+                               s.enlarge(s.bytelen + i - _buffer_pos)
 
                                # Copy from the buffer to the string
                                var j = _buffer_pos
@@ -664,5 +664,5 @@ class StringReader
                return new Bytes(nns, nslen, nslen)
        end
 
-       redef fun eof do return cursor >= source.length
+       redef fun eof do return cursor >= source.bytelen
 end
index cad7522..97191d2 100644 (file)
@@ -45,12 +45,14 @@ abstract class Text
        #
        #     assert "12345".length == 5
        #     assert "".length == 0
+       #     assert "あいうえお".length == 5
        fun length: Int is abstract
 
        # Number of bytes in `self`
        #
-       # TODO: Implement correctly once UTF-8 is supported
-       fun bytelen: Int do return length
+       #     assert "12345".bytelen == 5
+       #     assert "あいうえお".bytelen == 15
+       fun bytelen: Int is abstract
 
        # Create a substring.
        #
@@ -58,6 +60,7 @@ abstract class Text
        #     assert "abcd".substring(-1, 2)     ==  "a"
        #     assert "abcd".substring(1, 0)      ==  ""
        #     assert "abcd".substring(2, 5)      ==  "cd"
+       #     assert "あいうえお".substring(1,3) ==  "いうえ"
        #
        # A `from` index < 0 will be replaced by 0.
        # Unless a `count` value is > 0 at the same time.
@@ -934,7 +937,7 @@ abstract class FlatText
        # Real items, used as cache for to_cstring is called
        private var real_items: nullable NativeString = null
 
-       # Returns a char* starting at position `index_from`
+       # Returns a char* starting at position `first_byte`
        #
        # WARNING: If you choose to use this service, be careful of the following.
        #
@@ -953,6 +956,8 @@ abstract class FlatText
 
        redef var length = 0
 
+       redef var bytelen = 0
+
        redef fun output
        do
                var i = 0
@@ -1000,7 +1005,7 @@ private abstract class StringByteView
 
        redef fun iterator do return self.iterator_from(0)
 
-       redef fun reverse_iterator do return self.reverse_iterator_from(self.length - 1)
+       redef fun reverse_iterator do return self.reverse_iterator_from(target.bytelen - 1)
 end
 
 # Immutable sequence of characters.
@@ -1329,7 +1334,6 @@ private abstract class BufferByteView
        super Sequence[Byte]
 
        redef type SELFTYPE: Buffer
-
 end
 
 redef class Object
@@ -1574,6 +1578,8 @@ redef class Char
        #     assert '9'.is_numeric
        #     assert not 'a'.is_numeric
        #     assert not '?'.is_numeric
+       #
+       # FIXME: Works on ASCII-range only
        fun is_numeric: Bool
        do
                return self >= '0' and self <= '9'
@@ -1585,6 +1591,8 @@ redef class Char
        #     assert 'Z'.is_alpha
        #     assert not '0'.is_alpha
        #     assert not '?'.is_alpha
+       #
+       # FIXME: Works on ASCII-range only
        fun is_alpha: Bool
        do
                return (self >= 'a' and self <= 'z') or (self >= 'A' and self <= 'Z')
@@ -1597,6 +1605,8 @@ redef class Char
        #     assert '0'.is_alphanumeric
        #     assert '9'.is_alphanumeric
        #     assert not '?'.is_alphanumeric
+       #
+       # FIXME: Works on ASCII-range only
        fun is_alphanumeric: Bool
        do
                return self.is_numeric or self.is_alpha
index 66e7734..26c1a90 100644 (file)
@@ -12,6 +12,7 @@
 module flat
 
 intrude import abstract_text
+intrude import native
 
 `{
 #include <stdio.h>
@@ -38,28 +39,73 @@ class FlatString
        super FlatText
        super String
 
-       # Index in _items of the start of the string
-       private var index_from: Int is noinit
+       # Index at which `self` begins in `items`, inclusively
+       private var first_byte: Int is noinit
 
-       # Indes in _items of the last item of the string
-       private var index_to: Int is noinit
+       # Index at which `self` ends in `items`, inclusively
+       private var last_byte: Int is noinit
 
        redef var chars = new FlatStringCharView(self) is lazy
 
        redef var bytes = new FlatStringByteView(self) is lazy
 
-       redef fun [](index)
-       do
-               # Check that the index (+ index_from) is not larger than indexTo
-               # In other terms, if the index is valid
-               assert index >= 0
-               assert (index + index_from) <= index_to
-               return items[index + index_from].to_i.ascii
+       # Cache of the latest position (char) explored in the string
+       var position: Int = 0
+       # Cached position (bytes) in the NativeString underlying the String
+       var bytepos: Int = first_byte is lateinit
+
+       redef var length is lazy do
+               if bytelen == 0 then return 0
+               var st = first_byte
+               var its = items
+               var ln = 0
+               var lst = last_byte
+               while st <= lst do
+                       st += its.length_of_char_at(st)
+                       ln += 1
+               end
+               return ln
        end
 
-       ################################################
-       #       AbstractString specific methods        #
-       ################################################
+       redef fun [](index) do return items.char_at(char_to_byte_index(index))
+
+       # Index of the character `index` in `items`
+       private fun char_to_byte_index(index: Int): Int do
+               var ln = length
+               assert index >= 0
+               assert index < ln
+
+               # Find best insertion point
+               var delta_begin = index
+               var delta_end = (ln - 1) - index
+               var delta_cache = (position - index).abs
+               var min = delta_begin
+               var its = items
+
+               if delta_cache < min then min = delta_cache
+               if delta_end < min then min = delta_end
+
+               var ns_i: Int
+               var my_i: Int
+
+               if min == delta_begin then
+                       ns_i = first_byte
+                       my_i = 0
+               else if min == delta_cache then
+                       ns_i = bytepos
+                       my_i = position
+               else
+                       ns_i = its.find_beginning_of_char_at(last_byte)
+                       my_i = length - 1
+               end
+
+               ns_i = its.char_to_byte_index_cached(index, my_i, ns_i)
+
+               position = index
+               bytepos = ns_i
+
+               return ns_i
+       end
 
        redef fun reversed
        do
@@ -72,7 +118,7 @@ class FlatString
                return s
        end
 
-       redef fun fast_cstring do return items.fast_cstring(index_from)
+       redef fun fast_cstring do return items.fast_cstring(first_byte)
 
        redef fun substring(from, count)
        do
@@ -84,19 +130,16 @@ class FlatString
                        from = 0
                end
 
-               var new_from = index_from + from
+               if (count + from) > length then count = length - from
+               if count <= 0 then return ""
+               var end_index = from + count - 1
 
-               if (new_from + count) > index_to then
-                       var new_len = index_to - new_from + 1
-                       if new_len <= 0 then return empty
-                       return new FlatString.with_infos(items, new_len, new_from, index_to)
-               end
+               var bytefrom = char_to_byte_index(from)
+               var byteto = char_to_byte_index(end_index)
+               byteto += items.length_of_char_at(byteto) - 1
 
-               if count <= 0 then return empty
-
-               var to = new_from + count - 1
-
-               return new FlatString.with_infos(items, to - new_from + 1, new_from, to)
+               var s = new FlatString.full(items, byteto - bytefrom + 1, bytefrom, byteto, count)
+               return s
        end
 
        redef fun empty do return "".as(FlatString)
@@ -140,29 +183,38 @@ class FlatString
        #              String Specific Methods           #
        ##################################################
 
-       # Low-level creation of a new string with given data.
+       # Low-level creation of a new string with minimal data.
        #
        # `items` will be used as is, without copy, to retrieve the characters of the string.
        # Aliasing issues is the responsibility of the caller.
-       private init with_infos(items: NativeString, length: Int, from: Int, to: Int)
+       private init with_infos(items: NativeString, bytelen, from, to: Int)
        do
                self.items = items
-               self.length = length
-               index_from = from
-               index_to = to
+               self.bytelen = bytelen
+               first_byte = from
+               last_byte = to
        end
 
-       redef fun to_cstring
+       # Low-level creation of a new string with all the data.
+       #
+       # `items` will be used as is, without copy, to retrieve the characters of the string.
+       # Aliasing issues is the responsibility of the caller.
+       private init full(items: NativeString, bytelen, from, to, length: Int)
        do
-               if real_items != null then
-                       return real_items.as(not null)
-               else
-                       var newItems = new NativeString(length + 1)
-                       self.items.copy_to(newItems, length, index_from, 0)
-                       newItems[length] = 0u8
-                       self.real_items = newItems
-                       return newItems
-               end
+               self.items = items
+               self.length = length
+               self.bytelen = bytelen
+               first_byte = from
+               last_byte = to
+       end
+
+       redef fun to_cstring do
+               if real_items != null then return real_items.as(not null)
+               var new_items = new NativeString(bytelen + 1)
+               self.items.copy_to(new_items, bytelen, first_byte, 0)
+               new_items[bytelen] = 0u8
+               real_items = new_items
+               return new_items
        end
 
        redef fun ==(other)
@@ -171,12 +223,12 @@ class FlatString
 
                if self.object_id == other.object_id then return true
 
-               var my_length = length
+               var my_length = bytelen
 
-               if other.length != my_length then return false
+               if other.bytelen != my_length then return false
 
-               var my_index = index_from
-               var its_index = other.index_from
+               var my_index = first_byte
+               var its_index = other.first_byte
 
                var last_iteration = my_index + my_length
 
@@ -198,33 +250,22 @@ class FlatString
 
                if self.object_id == other.object_id then return false
 
-               var my_curr_char : Char
-               var its_curr_char : Char
-
-               var my_length = self.length
-               var its_length = other.length
-               var max
+               var my_length = self.bytelen
+               var its_length = other.bytelen
 
-               if my_length < its_length then
-                       max = my_length
-               else
-                       max = its_length
-               end
+               var max = if my_length < its_length then my_length else its_length
 
-               var my_chars = chars
-               var its_chars = other.chars
+               var myits = self.bytes
+               var itsits = other.bytes
 
-               var pos = 0
-               while pos < max do
-                       my_curr_char = my_chars[pos]
-                       its_curr_char = its_chars[pos]
+               for i in [0 .. max[ do
+                       var my_curr_char = myits[i]
+                       var its_curr_char = itsits[i]
 
                        if my_curr_char != its_curr_char then
                                if my_curr_char < its_curr_char then return true
                                return false
                        end
-
-                       pos += 1
                end
 
                return my_length < its_length
@@ -243,7 +284,7 @@ class FlatString
                        var ns = new NativeString(nlen + 1)
                        mits.copy_to(ns, mlen, mifrom, 0)
                        sits.copy_to(ns, slen, sifrom, mlen)
-                       return ns.to_s_with_length(nlen)
+                       return new FlatString.full(ns, nlen, 0, nlen - 1, length + o.length)
                else
                        abort
                end
@@ -265,16 +306,17 @@ class FlatString
                return new FlatString.full(ns, new_bytelen, 0, new_bytelen - 1, newlen)
        end
 
+
        redef fun hash
        do
                if hash_cache == null then
                        # djb2 hash algorithm
                        var h = 5381
-                       var i = index_from
+                       var i = first_byte
 
                        var myitems = items
 
-                       while i <= index_to do
+                       while i <= last_byte do
                                h = h.lshift(5) + h + myitems[i].to_i
                                i += 1
                        end
@@ -358,16 +400,16 @@ private class FlatStringByteReverseIterator
 
        init with_pos(tgt: FlatString, pos: Int)
        do
-               init(tgt, tgt.items, pos + tgt.index_from)
+               init(tgt, tgt.items, pos + tgt.first_byte)
        end
 
-       redef fun is_ok do return curr_pos >= target.index_from
+       redef fun is_ok do return curr_pos >= target.first_byte
 
        redef fun item do return target_items[curr_pos]
 
        redef fun next do curr_pos -= 1
 
-       redef fun index do return curr_pos - target.index_from
+       redef fun index do return curr_pos - target.first_byte
 
 end
 
@@ -382,16 +424,16 @@ private class FlatStringByteIterator
 
        init with_pos(tgt: FlatString, pos: Int)
        do
-               init(tgt, tgt.items, pos + tgt.index_from)
+               init(tgt, tgt.items, pos + tgt.first_byte)
        end
 
-       redef fun is_ok do return curr_pos <= target.index_to
+       redef fun is_ok do return curr_pos <= target.last_byte
 
        redef fun item do return target_items[curr_pos]
 
        redef fun next do curr_pos += 1
 
-       redef fun index do return curr_pos - target.index_from
+       redef fun index do return curr_pos - target.first_byte
 
 end
 
@@ -402,12 +444,12 @@ private class FlatStringByteView
 
        redef fun [](index)
        do
-               # Check that the index (+ index_from) is not larger than indexTo
+               # Check that the index (+ first_byte) is not larger than last_byte
                # In other terms, if the index is valid
                assert index >= 0
                var target = self.target
-               assert (index + target.index_from) <= target.index_to
-               return target.items[index + target.index_from]
+               assert (index + target.first_byte) <= target.last_byte
+               return target.items[index + target.first_byte]
        end
 
        redef fun iterator_from(start) do return new FlatStringByteIterator.with_pos(target, start)
@@ -431,7 +473,23 @@ class FlatBuffer
 
        redef var bytes: Sequence[Byte] = new FlatBufferByteView(self) is lazy
 
-       private var capacity: Int = 0
+       redef var bytelen = 0
+
+       # O(n)
+       redef fun length do
+               var max = bytelen
+               if max == 0 then return 0
+               var pos = 0
+               var ln = 0
+               var its = items
+               while pos < max do
+                       pos += its.length_of_char_at(pos)
+                       ln += 1
+               end
+               return ln
+       end
+
+       private var capacity = 0
 
        redef fun fast_cstring do return items.fast_cstring(0)
 
@@ -443,49 +501,84 @@ class FlatBuffer
        # the Copy-On-Write flag `written` is set at true.
        private fun reset do
                var nns = new NativeString(capacity)
-               items.copy_to(nns, length, 0, 0)
+               items.copy_to(nns, bytelen, 0, 0)
                items = nns
                written = false
        end
 
-       redef fun [](index)
+       # Shifts the content of the buffer by `len` bytes to the right, starting at byte `from`
+       #
+       # Internal only, does not modify bytelen or length, this is the caller's responsability
+       private fun rshift_bytes(from: Int, len: Int) do
+               var oit = items
+               var nit = items
+               if bytelen + len > capacity then
+                       capacity = capacity * 2 + 2
+                       nit = new NativeString(capacity)
+                       oit.copy_to(nit, 0, 0, from)
+               end
+               oit.copy_to(nit, bytelen - from, from, from + len)
+       end
+
+       # Shifts the content of the buffer by `len` bytes to the left, starting at `from`
+       #
+       # Internal only, does not modify bytelen or length, this is the caller's responsability
+       private fun lshift_bytes(from: Int, len: Int) do
+               items.copy_to(items, bytelen - from, from, from - len)
+       end
+
+       redef fun [](i)
        do
-               assert index >= 0
-               assert index < length
-               return items[index].to_i.ascii
+               assert i < length and i >= 0
+               return items.char_at(items.char_to_byte_index(i))
        end
 
        redef fun []=(index, item)
        do
+               assert index >= 0 and index <= length
+               if written then reset
                is_dirty = true
                if index == length then
-                       add(item)
+                       add item
                        return
                end
-               if written then reset
-               assert index >= 0 and index < length
-               items[index] = item.ascii.to_b
+               var ip = items.char_to_byte_index(index)
+               var c = items.char_at(ip)
+               var clen = c.u8char_len
+               var itemlen = item.u8char_len
+               var size_diff = itemlen - clen
+               if size_diff > 0 then
+                       rshift_bytes(ip + clen, size_diff)
+               else if size_diff < 0 then
+                       lshift_bytes(ip + clen, -size_diff)
+               end
+               bytelen += size_diff
+               items.set_char_at(ip, item)
        end
 
        redef fun add(c)
        do
+               if written then reset
                is_dirty = true
-               if capacity <= length then enlarge(length + 5)
-               items[length] = c.ascii.to_b
-               length += 1
+               var clen = c.u8char_len
+               enlarge(bytelen + clen)
+               items.set_char_at(bytelen, c)
+               bytelen += clen
        end
 
        private fun add_byte(b: Byte) do
+               if written then reset
                is_dirty = true
-               if capacity <= length then enlarge(length + 5)
+               enlarge(bytelen + 1)
                items[bytelen] = b
-               length += 1
+               # FIXME: Might trigger errors
+               bytelen += 1
        end
 
        redef fun clear do
                is_dirty = true
                if written then reset
-               length = 0
+               bytelen = 0
        end
 
        redef fun empty do return new Buffer
@@ -499,7 +592,7 @@ class FlatBuffer
                # it does a copy of the current `Buffer`
                written = false
                var a = new NativeString(c+1)
-               if length > 0 then items.copy_to(a, length, 0, 0)
+               if bytelen > 0 then items.copy_to(a, bytelen, 0, 0)
                items = a
                capacity = c
        end
@@ -507,16 +600,16 @@ class FlatBuffer
        redef fun to_s
        do
                written = true
-               if length == 0 then items = new NativeString(1)
-               return new FlatString.with_infos(items, length, 0, length - 1)
+               if bytelen == 0 then items = new NativeString(1)
+               return new FlatString.with_infos(items, bytelen, 0, bytelen - 1)
        end
 
        redef fun to_cstring
        do
                if is_dirty then
-                       var new_native = new NativeString(length + 1)
-                       new_native[length] = 0u8
-                       if length > 0 then items.copy_to(new_native, length, 0, 0)
+                       var new_native = new NativeString(bytelen + 1)
+                       new_native[bytelen] = 0u8
+                       if length > 0 then items.copy_to(new_native, bytelen, 0, 0)
                        real_items = new_native
                        is_dirty = false
                end
@@ -533,59 +626,51 @@ class FlatBuffer
        #
        # If `items` is shared, `written` should be set to true after the creation
        # so that a modification will do a copy-on-write.
-       private init with_infos(items: NativeString, capacity, length: Int)
+       private init with_infos(items: NativeString, capacity, bytelen: Int)
        do
                self.items = items
-               self.length = length
                self.capacity = capacity
+               self.bytelen = bytelen
        end
 
        # Create a new string copied from `s`.
        init from(s: Text)
        do
-               capacity = s.length + 1
-               length = s.length
-               items = new NativeString(capacity)
-               if s isa FlatString then
-                       s.items.copy_to(items, length, s.index_from, 0)
-               else if s isa FlatBuffer then
-                       s.items.copy_to(items, length, 0, 0)
+               items = new NativeString(s.bytelen)
+               if s isa FlatText then
+                       items = s.items
                else
-                       var curr_pos = 0
-                       for i in s.bytes do
-                               items[curr_pos] = i
-                               curr_pos += 1
-                       end
+                       for i in substrings do i.as(FlatString).items.copy_to(items, i.bytelen, 0, 0)
                end
+               bytelen = s.bytelen
+               capacity = s.bytelen
+               written = true
        end
 
        # Create a new empty string with a given capacity.
        init with_capacity(cap: Int)
        do
                assert cap >= 0
-               items = new NativeString(cap+1)
+               items = new NativeString(cap + 1)
                capacity = cap
-               length = 0
+               bytelen = 0
        end
 
        redef fun append(s)
        do
                if s.is_empty then return
                is_dirty = true
-               var sl = s.length
-               if capacity < length + sl then enlarge(length + sl)
+               var sl = s.bytelen
+               enlarge(bytelen + sl)
                if s isa FlatString then
-                       s.items.copy_to(items, sl, s.index_from, length)
+                       s.items.copy_to(items, sl, s.first_byte, bytelen)
                else if s isa FlatBuffer then
-                       s.items.copy_to(items, sl, 0, length)
+                       s.items.copy_to(items, sl, 0, bytelen)
                else
-                       var curr_pos = self.length
-                       for i in s.bytes do
-                               items[curr_pos] = i
-                               curr_pos += 1
-                       end
+                       for i in s.substrings do append i
+                       return
                end
-               length += sl
+               bytelen += sl
        end
 
        # Copies the content of self in `dest`
@@ -601,15 +686,16 @@ class FlatBuffer
        redef fun substring(from, count)
        do
                assert count >= 0
-               count += from
                if from < 0 then from = 0
-               if count > length then count = length
-               if from < count then
-                       var len = count - from
-                       var r_items = new NativeString(len)
-                       items.copy_to(r_items, len, from, 0)
-                       var r = new FlatBuffer.with_infos(r_items, len, len)
-                       return r
+               if (from + count) > length then count = length - from
+               if count != 0 then
+                       var bytefrom = items.char_to_byte_index(from)
+                       var byteto = items.char_to_byte_index(count + from - 1)
+                       byteto += items.char_at(byteto).u8char_len - 1
+                       var byte_length = byteto - bytefrom + 1
+                       var r_items = new NativeString(byte_length)
+                       items.copy_to(r_items, byte_length, bytefrom, 0)
+                       return new FlatBuffer.with_infos(r_items, byte_length, byte_length)
                else
                        return new Buffer
                end
@@ -618,22 +704,15 @@ class FlatBuffer
        redef fun reverse
        do
                written = false
-               var ns = new NativeString(capacity)
-               var si = length - 1
-               var ni = 0
-               var it = items
-               while si >= 0 do
-                       ns[ni] = it[si]
-                       ni += 1
-                       si -= 1
-               end
-               items = ns
+               var ns = new FlatBuffer.with_capacity(capacity)
+               for i in chars.reverse_iterator do ns.add i
+               items = ns.items
        end
 
        redef fun times(repeats)
        do
-               var x = new FlatString.with_infos(items, length, 0, length - 1)
-               for i in [1..repeats[ do
+               var x = new FlatString.with_infos(items, bytelen, 0, bytelen - 1)
+               for i in [1 .. repeats[ do
                        append(x)
                end
        end
@@ -641,21 +720,13 @@ class FlatBuffer
        redef fun upper
        do
                if written then reset
-               var id = length - 1
-               while id >= 0 do
-                       self[id] = self[id].to_upper
-                       id -= 1
-               end
+               for i in [0 .. length[ do self[i] = self[i].to_upper
        end
 
        redef fun lower
        do
                if written then reset
-               var id = length - 1
-               while id >= 0 do
-                       self[id] = self[id].to_lower
-                       id -= 1
-               end
+               for i in [0 .. length[ do self[i] = self[i].to_lower
        end
 end
 
@@ -745,7 +816,7 @@ private class FlatBufferByteIterator
 
        redef fun index do return curr_pos
 
-       redef fun is_ok do return curr_pos < target.length
+       redef fun is_ok do return curr_pos < target.bytelen
 
        redef fun item do return target_items[curr_pos]
 
@@ -925,7 +996,7 @@ redef class Int
                var ns = new NativeString(nslen + 1)
                ns[nslen] = 0u8
                native_int_to_s(ns, nslen + 1)
-               return ns.to_s_with_length(nslen)
+               return new FlatString.full(ns, nslen, 0, nslen - 1, nslen)
        end
 end
 
@@ -949,7 +1020,7 @@ redef class Array[E]
                                continue
                        end
                        var tmp = itsi.to_s
-                       sl += tmp.length
+                       sl += tmp.bytelen
                        na[mypos] = tmp
                        i += 1
                        mypos += 1
@@ -960,15 +1031,15 @@ redef class Array[E]
                var off = 0
                while i < mypos do
                        var tmp = na[i]
-                       var tpl = tmp.length
                        if tmp isa FlatString then
-                               tmp.items.copy_to(ns, tpl, tmp.index_from, off)
+                               var tpl = tmp.bytelen
+                               tmp.items.copy_to(ns, tpl, tmp.first_byte, off)
                                off += tpl
                        else
                                for j in tmp.substrings do
                                        var s = j.as(FlatString)
-                                       var slen = s.length
-                                       s.items.copy_to(ns, slen, s.index_from, off)
+                                       var slen = s.bytelen
+                                       s.items.copy_to(ns, slen, s.first_byte, off)
                                        off += slen
                                end
                        end
@@ -987,7 +1058,7 @@ redef class NativeArray[E]
                var sl = 0
                var mypos = 0
                while i < l do
-                       sl += na[i].length
+                       sl += na[i].bytelen
                        i += 1
                        mypos += 1
                end
@@ -997,15 +1068,15 @@ redef class NativeArray[E]
                var off = 0
                while i < mypos do
                        var tmp = na[i]
-                       var tpl = tmp.length
                        if tmp isa FlatString then
-                               tmp.items.copy_to(ns, tpl, tmp.index_from, off)
+                               var tpl = tmp.bytelen
+                               tmp.items.copy_to(ns, tpl, tmp.first_byte, off)
                                off += tpl
                        else
                                for j in tmp.substrings do
                                        var s = j.as(FlatString)
-                                       var slen = s.length
-                                       s.items.copy_to(ns, slen, s.index_from, off)
+                                       var slen = s.bytelen
+                                       s.items.copy_to(ns, slen, s.first_byte, off)
                                        off += slen
                                end
                        end
index 5d5878b..018217e 100644 (file)
@@ -76,18 +76,20 @@ private class Concat
 
        redef var length is noinit
 
+       redef var bytelen is noinit
+
        redef fun substrings do return new RopeSubstrings(self)
 
        redef fun empty do return ""
 
        redef var to_cstring is lazy do
-               var len = length
+               var len = bytelen
                var ns = new NativeString(len + 1)
                ns[len] = 0u8
                var off = 0
                for i in substrings do
-                       var ilen = i.length
-                       i.as(FlatString).items.copy_to(ns, ilen, i.as(FlatString).index_from, off)
+                       var ilen = i.bytelen
+                       i.as(FlatString).items.copy_to(ns, ilen, i.as(FlatString).first_byte, off)
                        off += ilen
                end
                return ns
@@ -100,6 +102,7 @@ private class Concat
 
        init do
                length = left.length + right.length
+               bytelen = left.bytelen + right.bytelen
        end
 
        redef fun output do
@@ -147,12 +150,12 @@ private class Concat
 
        redef fun +(o) do
                var s = o.to_s
-               var slen = s.length
+               var slen = s.bytelen
                if s isa Concat then
                        return new Concat(self, s)
                else
                        var r = right
-                       var rlen = r.length
+                       var rlen = r.bytelen
                        if rlen + slen > maxlen then return new Concat(self, s)
                        return new Concat(left, r + s)
                end
@@ -202,7 +205,7 @@ class RopeBuffer
        redef var bytes: Sequence[Byte] is lazy do return new RopeBufferBytes(self)
 
        # The final string being built on the fly
-       private var str: String is noinit
+       private var str: String = ""
 
        # Current concatenation buffer
        private var ns: NativeString is noinit
@@ -217,10 +220,21 @@ class RopeBuffer
        # a long string (length > maxlen) is appended.
        private var dumped: Int is noinit
 
-       # Length of the complete rope
-       redef var length = 0
+       # Length of the complete rope in chars (0)
+       redef fun length do
+               var st = dumped
+               var len = str.length
+               while st < rpos do
+                       st += ns[st].u8len
+                       len += 1
+               end
+               return len
+       end
+
+       # Length of the complete rope in bytes
+       redef var bytelen = 0
 
-       # Length of the mutable part
+       # Length of the mutable part (in bytes)
        #
        # Is also used as base to compute the size of the next
        # mutable native string (`ns`)
@@ -230,7 +244,6 @@ class RopeBuffer
 
        # Builds an empty `RopeBuffer`
        init do
-               str = ""
                ns = new NativeString(maxlen)
                buf_size = maxlen
                dumped = 0
@@ -241,7 +254,7 @@ class RopeBuffer
                self.str = str
                ns = new NativeString(maxlen)
                buf_size = maxlen
-               length = str.length
+               bytelen = str.length
                dumped = 0
        end
 
@@ -306,7 +319,7 @@ class RopeBuffer
 
        redef fun clear do
                str = ""
-               length = 0
+               bytelen = 0
                rpos = 0
                dumped = 0
                if written then
@@ -347,63 +360,29 @@ class RopeBuffer
        end
 
        redef fun append(s) do
-               var slen = s.length
-               length += slen
-               var rp = rpos
-               if s isa Rope or slen > maxlen then
-                       if rp > 0 and dumped != rp then
-                               str += new FlatString.with_infos(ns, rp - dumped, dumped, rp - 1)
-                               dumped = rp
-                       end
-                       str = str + s
+               var slen = s.bytelen
+               if slen >= maxlen then
+                       persist_buffer
+                       str += s.to_s
                        return
                end
-               var remsp = buf_size - rp
-               var sits: NativeString
-               var begin: Int
-               if s isa FlatString then
-                       begin = s.index_from
-                       sits = s.items
-               else if s isa FlatBuffer then
-                       begin = 0
-                       sits = s.items
-               else
+               if s isa FlatText then
+                       var oits = s.items
+                       var from = if s isa FlatString then s.first_byte else 0
+                       var remsp = buf_size - rpos
                        if slen <= remsp then
-                               for i in s.bytes do
-                                       ns[rpos] = i
-                                       rpos += 1
-                               end
-                       else
-                               var spos = 0
-                               for i in [0..remsp[ do
-                                       ns[rpos] = s.bytes[spos]
-                                       rpos += 1
-                                       spos += 1
-                               end
-                               dump_buffer
-                               while spos < slen do
-                                       ns[rpos] = s.bytes[spos]
-                                       spos += 1
-                                       rpos += 1
-                               end
-                       end
-                       return
-               end
-               if slen <= remsp then
-                       if remsp <= 0 then
-                               dump_buffer
-                               rpos = 0
-                       else
-                               sits.copy_to(ns, slen, begin, rp)
+                               oits.copy_to(ns, slen, from, rpos)
                                rpos += slen
+                               return
                        end
-               else
-                       sits.copy_to(ns, remsp, begin, rp)
-                       rpos = buf_size
+                       var brk = oits.find_beginning_of_char_at(from + remsp)
+                       oits.copy_to(ns, brk, from, rpos)
+                       rpos += brk
                        dump_buffer
-                       var nlen = slen - remsp
-                       sits.copy_to(ns, nlen, begin + remsp, 0)
-                       rpos = nlen
+                       oits.copy_to(ns, slen - remsp, brk, 0)
+                       rpos = slen - remsp
+               else
+                       for i in s.substrings do append i
                end
        end
 
@@ -416,7 +395,7 @@ class RopeBuffer
                # TODO: Fix when supporting UTF-8
                ns[rp] = c.ascii.to_b
                rp += 1
-               length += 1
+               bytelen += 1
                rpos = rp
        end
 
@@ -428,7 +407,7 @@ class RopeBuffer
                end
                ns[rp] = b
                rp += 1
-               length += 1
+               bytelen += 1
                rpos = rp
        end
 
@@ -443,10 +422,12 @@ class RopeBuffer
                ns = new NativeString(bs)
                buf_size = bs
                dumped = 0
+               rpos = 0
        end
 
        # Similar to dump_buffer, but does not reallocate a new NativeString
        private fun persist_buffer do
+               if rpos == dumped then return
                var nstr = new FlatString.with_infos(ns, rpos - dumped, dumped, rpos - 1)
                str += nstr
                dumped = rpos
@@ -468,10 +449,8 @@ class RopeBuffer
        redef fun enlarge(i) do end
 
        redef fun to_s do
-               written = true
-               var nnslen = rpos - dumped
-               if nnslen == 0 then return str
-               return str + new FlatString.with_infos(ns, rpos - dumped, dumped, rpos - 1)
+               dump_buffer
+               return str
        end
 
        redef fun reverse do
@@ -506,16 +485,16 @@ redef class FlatString
 
        redef fun +(o) do
                var s = o.to_s
-               var slen = s.length
-               var mlen = length
+               var slen = s.bytelen
+               var mlen = bytelen
                if slen == 0 then return self
                if mlen == 0 then return s
                var nlen = slen + mlen
                if s isa FlatString then
                        if nlen > maxlen then return new Concat(self, s)
                        var mits = items
-                       var sifrom = s.index_from
-                       var mifrom = index_from
+                       var sifrom = s.first_byte
+                       var mifrom = first_byte
                        var sits = s.items
                        var ns = new NativeString(nlen + 1)
                        mits.copy_to(ns, mlen, mifrom, 0)
@@ -523,7 +502,7 @@ redef class FlatString
                        return ns.to_s_with_length(nlen)
                else if s isa Concat then
                        var sl = s.left
-                       var sllen = sl.length
+                       var sllen = sl.bytelen
                        if sllen + mlen > maxlen then return new Concat(self, s)
                        return new Concat(self + sl, s.right)
                else
@@ -559,11 +538,11 @@ private class RopeByteReverseIterator
        var subs: IndexedIterator[FlatString]
 
        init(root: Concat) is old_style_init do
-               pos = root.length - 1
+               pos = root.bytelen - 1
                subs = new ReverseRopeSubstrings(root)
                var s = subs.item
                ns = s.items
-               pns = s.index_to
+               pns = s.last_byte
        end
 
        init from(root: Concat, pos: Int) do
@@ -589,7 +568,7 @@ private class RopeByteReverseIterator
                if not subs.is_ok then return
                var s = subs.item
                ns = s.items
-               pns = s.index_to
+               pns = s.last_byte
        end
 end
 
@@ -633,7 +612,7 @@ private class RopeByteIterator
        redef fun next do
                pns += 1
                pos += 1
-               if pns < subs.item.length then return
+               if pns < subs.item.bytelen then return
                if not subs.is_ok then return
                subs.next
                if not subs.is_ok then return
@@ -975,7 +954,6 @@ private class RopeBytes
        redef type SELFTYPE: Concat
 
        redef fun [](i) do
-               var b: Int
                var nod: String = target
                loop
                        if nod isa FlatString then return nod.items[i]
@@ -1096,7 +1074,7 @@ class RopeBufferByteIterator
        # Init the iterator from a RopeBuffer.
        init(t: RopeBuffer) is old_style_init do
                ns = t.ns
-               maxpos = t.rpos
+               maxpos = t.bytelen
                sit = t.str.bytes.iterator
                pns = t.dumped
                index = 0
@@ -1105,7 +1083,7 @@ class RopeBufferByteIterator
        # Init the iterator from a RopeBuffer starting from `pos`.
        init from(t: RopeBuffer, pos: Int) do
                ns = t.ns
-               maxpos = t.length
+               maxpos = t.bytelen
                sit = t.str.bytes.iterator_from(pos)
                pns = pos - t.str.length
                index = pos
@@ -1147,19 +1125,19 @@ class RopeBufferByteReverseIterator
        init(tgt: RopeBuffer) is old_style_init do
                sit = tgt.str.bytes.reverse_iterator
                pns = tgt.rpos - 1
-               index = tgt.length - 1
+               index = tgt.bytelen - 1
                ns = tgt.ns
        end
 
        # Init the iterator from a RopeBuffer starting from `pos`.
        init from(tgt: RopeBuffer, pos: Int) do
-               sit = tgt.str.bytes.reverse_iterator_from(pos - tgt.rpos - tgt.dumped)
-               pns = pos - tgt.str.length
+               sit = tgt.str.bytes.reverse_iterator_from(pos - (tgt.rpos - tgt.dumped))
+               pns = pos - tgt.str.bytelen + tgt.rpos
                index = pos
                ns = tgt.ns
        end
 
-       redef fun is_ok do return index > 0
+       redef fun is_ok do return index >= 0
 
        redef fun item do
                if pns >= 0 then return ns[pns]
@@ -1168,7 +1146,7 @@ class RopeBufferByteReverseIterator
 
        redef fun next do
                index -= 1
-               if pns >= 0 then
+               if pns > 0 then
                        pns -= 1
                else
                        sit.next
@@ -1186,7 +1164,7 @@ class RopeBufferBytes
                if i < target.str.bytelen then
                        return target.str.bytes[i]
                else
-                       return target.ns[i - target.str.length]
+                       return target.ns[i - target.str.bytelen]
                end
        end
 
index d384512..441df14 100644 (file)
@@ -137,10 +137,10 @@ class WebsocketConnection
                        ans_buffer.add(msg.length.to_b)
                end
                if msg isa FlatString then
-                       ans_buffer.append_ns_from(msg.items, msg.length, msg.index_from)
+                       ans_buffer.append_ns_from(msg.items, msg.length, msg.first_byte)
                else
                        for i in msg.substrings do
-                               ans_buffer.append_ns_from(i.as(FlatString).items, i.length, i.as(FlatString).index_from)
+                               ans_buffer.append_ns_from(i.as(FlatString).items, i.length, i.as(FlatString).first_byte)
                        end
                end
                return ans_buffer