lib/standard: NativeString operations are now Byte-oriented instead of Char
authorLucas Bajolet <r4pass@hotmail.com>
Wed, 8 Jul 2015 20:45:15 +0000 (16:45 -0400)
committerLucas Bajolet <r4pass@hotmail.com>
Thu, 9 Jul 2015 15:45:40 +0000 (11:45 -0400)
Signed-off-by: Lucas Bajolet <r4pass@hotmail.com>

lib/bitmap/bitmap.nit
lib/buffered_ropes.nit
lib/saxophonit/lexer.nit
lib/standard/bytes.nit
lib/standard/stream.nit
lib/standard/text/abstract_text.nit
lib/standard/text/flat.nit
lib/standard/text/native.nit
lib/standard/text/ropes.nit

index 003eb6f..505abcf 100644 (file)
@@ -133,7 +133,7 @@ class Bitmap
                        if b == null then
                                return
                        end
-                       bitmap_header[x] = b
+                       bitmap_header[x] = b.to_i
                end
                self.file_size = get_value(bitmap_header.subarray(2, 4))
                self.data_offset = get_value(bitmap_header.subarray(10, 4))
@@ -142,7 +142,7 @@ class Bitmap
                for x in [0..39] do
                        var b = fileReader.read_byte
                        if b == null then return
-                       dib_header[x] = b
+                       dib_header[x] = b.to_i
                end
                var dib_size = get_value(dib_header.subarray(0, 4))
                # only support BITMAPINFOHEADER
@@ -176,7 +176,7 @@ class Bitmap
                                        var red = bts[0] << 16
                                        var green = bts[1] << 8
                                        var blue = bts[2]
-                                       row.add(red + green + blue)
+                                       row.add(red.to_i + green.to_i + blue.to_i)
                                end
                                self.data.add(row)
                        end
index 06b2415..a2a8023 100644 (file)
@@ -59,7 +59,7 @@ private class ManualBuffer
 
        init do ns = new NativeString(maxlen)
 
-       fun [](i: Int): Char do return ns[i]
+       fun [](i: Int): Byte do return ns[i]
 end
 
 # Simple implementation of the iterator on Substrings for `Leaf`
@@ -92,21 +92,21 @@ private class Leaf
 
        var buf: ManualBuffer
        var bns: NativeString is noinit
-       redef var length: Int is noinit
+       redef var length is noinit
 
        redef fun empty do return new Leaf(new ManualBuffer)
 
        redef fun to_cstring do
                var len = length
                var ns = new NativeString(len + 1)
-               ns[len] = '\0'
+               ns[len] = 0u8
                buf.ns.copy_to(ns, len, 0, 0)
                return ns
        end
 
        redef fun substrings do return new LeafSubstrings(self)
 
-       redef fun [](i) do return buf[i]
+       redef fun [](i) do return buf[i].to_i.ascii
 
        init do
                bns = buf.ns
@@ -116,27 +116,15 @@ private class Leaf
        redef fun output do new FlatString.with_infos(buf.ns, length, 0, length - 1).output
 
        redef fun to_upper do
-               var x = new ManualBuffer
-               var nns = x.ns
-               var ns = bns
-               var mlen = length
-               for i in [0..mlen[ do
-                       nns[i] = ns[i].to_upper
-               end
-               x.pos = mlen - 1
-               return new Leaf(x)
+               var x = new FlatBuffer
+               for i in chars do x.add(i.to_upper)
+               return x.to_s
        end
 
        redef fun to_lower do
-               var x = new ManualBuffer
-               var nns = x.ns
-               var ns = bns
-               var mlen = length
-               for i in [0..mlen[ do
-                       nns[i] = ns[i].to_lower
-               end
-               x.pos = mlen - 1
-               return new Leaf(x)
+               var x = new FlatBuffer
+               for i in chars do x.add(i.to_lower)
+               return x.to_s
        end
 
        redef fun reversed do
@@ -165,8 +153,8 @@ private class Leaf
 
        redef fun +(o) do
                var s = o.to_s
-               var slen = s.length
-               var mlen = length
+               var slen = s.bytelen
+               var mlen = bytelen
                if slen == 0 then return self
                if mlen == 0 then return s
                var nlen = mlen + slen
@@ -213,7 +201,7 @@ private class Leaf
                                b = new ManualBuffer
                                bns.copy_to(b.ns, mlen, 0, 0)
                        end
-                       for i in s.chars do
+                       for i in s.bytes do
                                bns[bpos] = i
                                bpos += 1
                        end
@@ -226,7 +214,7 @@ redef class Concat
        redef fun to_cstring do
                var len = length
                var ns = new NativeString(len + 1)
-               ns[len] = '\0'
+               ns[len] = 0u8
                var off = 0
                for i in substrings do
                        var ilen = i.length
@@ -328,7 +316,7 @@ redef class Array[E]
                        mypos += 1
                end
                var ns = new NativeString(sl + 1)
-               ns[sl] = '\0'
+               ns[sl] = 0u8
                i = 0
                var off = 0
                while i < mypos do
index a84802d..acfe9fa 100644 (file)
@@ -259,7 +259,7 @@ class XophonLexer
                        last_char = -1
                        return
                end
-               last_char = s
+               last_char = s.to_i
 
                # XML 1.0 end-of-line handling
                # Note: Regardless the XML version, any EOL defined by the
@@ -267,8 +267,11 @@ class XophonLexer
                if was_cr and last_char == '\n'.ascii then
                        # EOL already reported. => Skip this byte.
                        s = input.read_byte
-                       if s == null then s = -1
-                       last_char = s
+                       if s == null then
+                               last_char = -1
+                       else
+                               last_char = s.to_i
+                       end
                end
                was_cr = last_char == '\r'.ascii
                if was_cr then
index c02ff00..2a3526f 100644 (file)
@@ -22,10 +22,8 @@ intrude import text::flat
 # A buffer containing Byte-manipulation facilities
 #
 # Uses Copy-On-Write when persisted
-#
-# TODO: Change the bound to Byte when available in stdlib and bootstrap
 class Bytes
-       super AbstractArray[Int]
+       super AbstractArray[Byte]
 
        # A NativeString being a char*, it can be used as underlying representation here.
        private var items: NativeString
@@ -56,39 +54,39 @@ class Bytes
        redef fun is_empty do return length != 0
 
        #     var b = new Bytes.empty
-       #     b.add 101
-       #     assert b[0] == 101
+       #     b.add 101u8
+       #     assert b[0] == 101u8
        redef fun [](i) do
                assert i >= 0
                assert i < length
-               return items[i].ascii
+               return items[i]
        end
 
        #     var b = new Bytes.with_capacity(1)
-       #     b[0] = 101
+       #     b[0] = 101u8
        #     assert b.to_s == "e"
        redef fun []=(i, v) do
                if persisted then regen
                assert i >= 0
                assert i <= length
                if i == length then add(v)
-               items[i] = v.ascii
+               items[i] = v
        end
 
        #     var b = new Bytes.empty
-       #     b.add 101
+       #     b.add 101u8
        #     assert b.to_s == "e"
        redef fun add(c) do
                if persisted then regen
                if length >= capacity then
                        enlarge(length)
                end
-               items[length] = c.ascii
+               items[length] = c
                length += 1
        end
 
        #     var b = new Bytes.empty
-       #     b.append([104, 101, 108, 108, 111])
+       #     b.append([104u8, 101u8, 108u8, 108u8, 111u8])
        #     assert b.to_s == "hello"
        redef fun append(arr) do
                if arr isa Bytes then
@@ -143,7 +141,7 @@ class Bytes
 end
 
 private class BytesIterator
-       super IndexedIterator[Int]
+       super IndexedIterator[Byte]
 
        var tgt: NativeString
 
@@ -157,7 +155,7 @@ private class BytesIterator
 
        redef fun next do index += 1
 
-       redef fun item do return tgt[index].ascii
+       redef fun item do return tgt[index]
 end
 
 redef class NativeString
index 0bfd4b0..dd1875b 100644 (file)
@@ -405,7 +405,8 @@ abstract class BufferedReader
                        last_error = new IOError("Stream has reached eof")
                        return null
                end
-               var c = _buffer[_buffer_pos]
+               # TODO: Fix when supporting UTF-8
+               var c = _buffer[_buffer_pos].to_i.ascii
                _buffer_pos += 1
                return c
        end
@@ -417,7 +418,7 @@ abstract class BufferedReader
                        last_error = new IOError("Stream has reached eof")
                        return null
                end
-               var c = _buffer[_buffer_pos].ascii
+               var c = _buffer[_buffer_pos]
                _buffer_pos += 1
                return c
        end
@@ -496,7 +497,7 @@ abstract class BufferedReader
                        var j = _buffer_pos
                        var k = _buffer_length
                        while j < k do
-                               s.add(_buffer[j].ascii)
+                               s.add(_buffer[j])
                                j += 1
                        end
                        _buffer_pos = j
@@ -510,13 +511,13 @@ abstract class BufferedReader
                loop
                        # First phase: look for a '\n'
                        var i = _buffer_pos
-                       while i < _buffer_length and _buffer[i] != '\n' do
+                       while i < _buffer_length and _buffer[i] != 0xAu8 do
                                i += 1
                        end
 
                        var eol
                        if i < _buffer_length then
-                               assert _buffer[i] == '\n'
+                               assert _buffer[i] == 0xAu8
                                i += 1
                                eol = true
                        else
@@ -531,7 +532,7 @@ abstract class BufferedReader
                                # Copy from the buffer to the string
                                var j = _buffer_pos
                                while j < i do
-                                       s.add(_buffer[j])
+                                       s.bytes.add(_buffer[j])
                                        j += 1
                                end
                                _buffer_pos = i
@@ -628,13 +629,13 @@ class StringReader
        # The string to read from.
        var source: String
 
-       # The current position in the string.
+       # The current position in the string (bytewise).
        private var cursor: Int = 0
 
        redef fun read_char do
                if cursor < source.length then
+                       # Fix when supporting UTF-8
                        var c = source[cursor]
-
                        cursor += 1
                        return c
                else
@@ -644,10 +645,9 @@ class StringReader
 
        redef fun read_byte do
                if cursor < source.length then
-                       var c = source[cursor]
-
+                       var c = source.bytes[cursor]
                        cursor += 1
-                       return c.ascii
+                       return c
                else
                        return null
                end
index e0f0f96..93d613f 100644 (file)
@@ -912,7 +912,7 @@ abstract class Text
                var mypos = src_offset
                var itspos = dest_offset
                while n > 0 do
-                       dest[itspos] = self.chars[mypos]
+                       dest[itspos] = self.bytes[mypos]
                        itspos += 1
                        mypos += 1
                        n -= 1
@@ -1390,7 +1390,7 @@ redef class Byte
        redef fun to_s do
                var nslen = byte_to_s_len
                var ns = new NativeString(nslen + 1)
-               ns[nslen] = '\0'
+               ns[nslen] = 0u8
                native_byte_to_s(ns, nslen + 1)
                return ns.to_s_with_length(nslen)
        end
index 4826ee7..15ea14f 100644 (file)
@@ -54,7 +54,7 @@ class FlatString
                # In other terms, if the index is valid
                assert index >= 0
                assert (index + index_from) <= index_to
-               return items[index + index_from]
+               return items[index + index_from].to_i.ascii
        end
 
        ################################################
@@ -163,7 +163,7 @@ class FlatString
                else
                        var newItems = new NativeString(length + 1)
                        self.items.copy_to(newItems, length, index_from, 0)
-                       newItems[length] = '\0'
+                       newItems[length] = 0u8
                        self.real_items = newItems
                        return newItems
                end
@@ -256,7 +256,7 @@ class FlatString
                        end
                end
 
-               target_string[total_length] = '\0'
+               target_string[total_length] = 0u8
 
                return target_string.to_s_with_length(total_length)
        end
@@ -273,7 +273,7 @@ class FlatString
 
                var target_string = new NativeString(final_length + 1)
 
-               target_string[final_length] = '\0'
+               target_string[final_length] = 0u8
 
                var current_last = 0
 
@@ -295,7 +295,7 @@ class FlatString
                        var myitems = items
 
                        while i <= index_to do
-                               h = h.lshift(5) + h + myitems[i].ascii
+                               h = h.lshift(5) + h + myitems[i].to_i
                                i += 1
                        end
 
@@ -478,8 +478,8 @@ class FlatBuffer
        redef fun [](index)
        do
                assert index >= 0
-               assert index  < length
-               return items[index]
+               assert index < length
+               return items[index].to_i.ascii
        end
 
        redef fun []=(index, item)
@@ -491,7 +491,7 @@ class FlatBuffer
                end
                if written then reset
                assert index >= 0 and index < length
-               items[index] = item
+               items[index] = item.ascii.to_b
        end
 
        redef fun add(c)
@@ -542,7 +542,7 @@ class FlatBuffer
        do
                if is_dirty then
                        var new_native = new NativeString(length + 1)
-                       new_native[length] = '\0'
+                       new_native[length] = 0u8
                        if length > 0 then items.copy_to(new_native, length, 0, 0)
                        real_items = new_native
                        is_dirty = false
@@ -579,9 +579,8 @@ class FlatBuffer
                        s.items.copy_to(items, length, 0, 0)
                else
                        var curr_pos = 0
-                       for i in [0..s.length[ do
-                               var c = s.chars[i]
-                               items[curr_pos] = c
+                       for i in s.bytes do
+                               items[curr_pos] = i
                                curr_pos += 1
                        end
                end
@@ -608,9 +607,8 @@ class FlatBuffer
                        s.items.copy_to(items, sl, 0, length)
                else
                        var curr_pos = self.length
-                       for i in [0..s.length[ do
-                               var c = s.chars[i]
-                               items[curr_pos] = c
+                       for i in s.bytes do
+                               items[curr_pos] = i
                                curr_pos += 1
                        end
                end
@@ -854,27 +852,27 @@ private class FlatBufferCharView
 
 end
 
-private class FlatBufferIterator
+private class FlatBufferCharIterator
        super IndexedIterator[Char]
 
        var target: FlatBuffer
 
-       var target_items: NativeString
+       var max: Int
 
        var curr_pos: Int
 
        init with_pos(tgt: FlatBuffer, pos: Int)
        do
                target = tgt
-               if tgt.length > 0 then target_items = tgt.items
+               max = tgt.length - 1
                curr_pos = pos
        end
 
        redef fun index do return curr_pos
 
-       redef fun is_ok do return curr_pos < target.length
+       redef fun is_ok do return curr_pos <= max
 
-       redef fun item do return target_items[curr_pos]
+       redef fun item do return target[curr_pos]
 
        redef fun next do curr_pos += 1
 
@@ -901,7 +899,7 @@ redef class NativeString
                var new_self = new NativeString(length + 1)
                copy_to(new_self, length, 0, 0)
                var str = new FlatString.with_infos(new_self, length, 0, length - 1)
-               new_self[length] = '\0'
+               new_self[length] = 0u8
                str.real_items = new_self
                return str
        end
@@ -927,7 +925,7 @@ redef class Int
 
                var nslen = int_to_s_len
                var ns = new NativeString(nslen + 1)
-               ns[nslen] = '\0'
+               ns[nslen] = 0u8
                native_int_to_s(ns, nslen + 1)
                return ns.to_s_with_length(nslen)
        end
@@ -959,7 +957,7 @@ redef class Array[E]
                        mypos += 1
                end
                var ns = new NativeString(sl + 1)
-               ns[sl] = '\0'
+               ns[sl] = 0u8
                i = 0
                var off = 0
                while i < mypos do
@@ -996,7 +994,7 @@ redef class NativeArray[E]
                        mypos += 1
                end
                var ns = new NativeString(sl + 1)
-               ns[sl] = '\0'
+               ns[sl] = 0u8
                i = 0
                var off = 0
                while i < mypos do
index c50caa0..8b66384 100644 (file)
@@ -25,10 +25,10 @@ extern class NativeString `{ char* `}
        fun fast_cstring(index: Int): NativeString is intern
 
        # Get char at `index`.
-       fun [](index: Int): Char is intern
+       fun [](index: Int): Byte is intern
 
        # Set char `item` at index.
-       fun []=(index: Int, item: Char) is intern
+       fun []=(index: Int, item: Byte) is intern
 
        # Copy `self` to `dest`.
        fun copy_to(dest: NativeString, length: Int, from: Int, to: Int) is intern
@@ -37,7 +37,7 @@ extern class NativeString `{ char* `}
        fun cstring_length: Int
        do
                var l = 0
-               while self[l] != '\0' do l += 1
+               while self[l] != 0u8 do l += 1
                return l
        end
 
index 4064812..7ee2bbd 100644 (file)
@@ -83,7 +83,7 @@ private class Concat
        redef var to_cstring is lazy do
                var len = length
                var ns = new NativeString(len + 1)
-               ns[len] = '\0'
+               ns[len] = 0u8
                var off = 0
                for i in substrings do
                        var ilen = i.length