X-Git-Url: http://nitlanguage.org diff --git a/lib/string_experimentations/utf8.nit b/lib/string_experimentations/utf8.nit index f65a359..b3a2450 100644 --- a/lib/string_experimentations/utf8.nit +++ b/lib/string_experimentations/utf8.nit @@ -16,6 +16,7 @@ module utf8 intrude import standard::string +intrude import standard::file in "C Header" `{ @@ -33,7 +34,7 @@ typedef struct { # UTF-8 char as defined in RFC-3629, e.g. 1-4 Bytes # # A UTF-8 char has its bytes stored in a NativeString (char*) -extern class UnicodeChar `{ UTF8Char* `} +extern class UTF8Char `{ UTF8Char* `} new(pos: Int, ns: NativeString) `{ UTF8Char* u = malloc(sizeof(UTF8Char)); @@ -46,6 +47,7 @@ extern class UnicodeChar `{ UTF8Char* `} # # As per the specification : # + # ~~~raw # Length | UTF-8 octet sequence # | (binary) # ---------+------------------------------------------------- @@ -53,9 +55,10 @@ extern class UnicodeChar `{ UTF8Char* `} # 2 | 110xxxxx 10xxxxxx # 3 | 1110xxxx 10xxxxxx 10xxxxxx # 4 | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + # ~~~ private fun len: Int `{ - char* ns = recv->ns; - int pos = recv->pos; + char* ns = self->ns; + int pos = self->pos; char nspos = ns[pos]; if((nspos & 0x80) == 0x00){ return 1;} if((nspos & 0xE0) == 0xC0){ return 2;} @@ -67,34 +70,34 @@ extern class UnicodeChar `{ UTF8Char* `} # Position in containing NativeString private fun pos: Int `{ - return recv->pos; + return self->pos; `} - private fun pos=(p: Int) `{recv->pos = p;`} + private fun pos=(p: Int) `{self->pos = p;`} # C char* wrapping the char fun ns: NativeString `{ - return recv->ns; + return self->ns; `} # Returns the Unicode code point representing the character # # Note : A unicode character might not be a visible glyph, but it will be used to determine canonical equivalence - fun code_point: Int import UnicodeChar.len `{ - switch(UnicodeChar_len(recv)){ + fun code_point: Int import UTF8Char.len `{ + switch(UTF8Char_len(self)){ case 1: - return (long)(0x7F & (unsigned char)recv->ns[recv->pos]); + return (long)(0x7F & (unsigned char)self->ns[self->pos]); case 2: - return 0 | ((0x1F & (unsigned char)recv->ns[recv->pos]) << 6) | (0x3F & (unsigned char)recv->ns[recv->pos+1]); + return 0 | ((0x1F & (unsigned char)self->ns[self->pos]) << 6) | (0x3F & (unsigned char)self->ns[self->pos+1]); case 3: - return 0 | ((0x0F & (unsigned char)recv->ns[recv->pos]) << 12) | - ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 6) | - (0x3F & (unsigned char)recv->ns[recv->pos+2]); + return 0 | ((0x0F & (unsigned char)self->ns[self->pos]) << 12) | + ((0x3F & (unsigned char)self->ns[self->pos+1]) << 6) | + (0x3F & (unsigned char)self->ns[self->pos+2]); case 4: - return 0 | ((0x07 & (unsigned char)recv->ns[recv->pos]) << 18) | - ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 12) | - ((0x3F & (unsigned char)recv->ns[recv->pos+2]) << 6) | - (0x3F & (unsigned char)recv->ns[recv->pos+3]); + return 0 | ((0x07 & (unsigned char)self->ns[self->pos]) << 18) | + ((0x3F & (unsigned char)self->ns[self->pos+1]) << 12) | + ((0x3F & (unsigned char)self->ns[self->pos+2]) << 6) | + (0x3F & (unsigned char)self->ns[self->pos+3]); } `} @@ -102,12 +105,12 @@ extern class UnicodeChar `{ UTF8Char* `} # # NOTE : Works only on ASCII chars # TODO : Support unicode for to_upper - fun to_upper: UnicodeChar import UnicodeChar.code_point `{ - int cp = UnicodeChar_code_point(recv); - if(cp < 97 || cp > 122){ return recv; } + fun to_upper: UTF8Char import UTF8Char.code_point `{ + int cp = UTF8Char_code_point(self); + if(cp < 97 || cp > 122){ return self; } char* ns = malloc(2); ns[1] = '\0'; - char c = recv->ns[recv->pos]; + char c = self->ns[self->pos]; ns[0] = c - 32; UTF8Char* ret = malloc(sizeof(UTF8Char)); ret->ns = ns; @@ -119,12 +122,12 @@ extern class UnicodeChar `{ UTF8Char* `} # # NOTE : Works only on ASCII chars # TODO : Support unicode for to_upper - fun to_lower: UnicodeChar import UnicodeChar.code_point `{ - int cp = UnicodeChar_code_point(recv); - if(cp < 65 || cp > 90){ return recv; } + fun to_lower: UTF8Char import UTF8Char.code_point `{ + int cp = UTF8Char_code_point(self); + if(cp < 65 || cp > 90){ return self; } char* ns = malloc(2); ns[1] = '\0'; - char c = recv->ns[recv->pos]; + char c = self->ns[self->pos]; ns[0] = c + 32; UTF8Char* ret = malloc(sizeof(UTF8Char)); ret->ns = ns; @@ -137,18 +140,35 @@ extern class UnicodeChar `{ UTF8Char* `} if o isa Char then if len != 1 then return false if code_point == o.ascii then return true - else if o isa UnicodeChar then + else if o isa UTF8Char then if len != o.len then return false if code_point == o.code_point then return true end return false end + redef fun output import UTF8Char.code_point `{ + switch(UTF8Char_len(self)){ + case 1: + printf("%c", self->ns[self->pos]); + break; + case 2: + printf("%c%c", self->ns[self->pos], self->ns[self->pos + 1]); + break; + case 3: + printf("%c%c%c", self->ns[self->pos], self->ns[self->pos + 1], self->ns[self->pos + 2]); + break; + case 4: + printf("%c%c%c%c", self->ns[self->pos], self->ns[self->pos + 1], self->ns[self->pos + 2], self->ns[self->pos + 3]); + break; + } + `} + redef fun to_s import NativeString.to_s_with_length `{ - int len = utf8___UnicodeChar_len___impl(recv); + int len = utf8___UTF8Char_len___impl(self); char* r = malloc(len + 1); r[len] = '\0'; - char* src = (recv->ns + recv->pos); + char* src = (self->ns + self->pos); memcpy(r, src, len); return NativeString_to_s_with_length(r, len); `} @@ -162,14 +182,14 @@ private extern class StringIndex `{ UTF8Char* `} new(size: Int) `{ return malloc(size*sizeof(UTF8Char)); `} # Sets the character at `index` as `item` - fun []=(index: Int, item: UnicodeChar) `{ recv[index] = *item; `} + fun []=(index: Int, item: UTF8Char) `{ self[index] = *item; `} # Gets the character at position `id` - fun [](id: Int): UnicodeChar `{ return &recv[id]; `} + fun [](id: Int): UTF8Char `{ return &self[id]; `} # Copies a part of self starting at index `my_from` of length `length` into `other`, starting at `its_from` fun copy_to(other: StringIndex, my_from: Int, its_from: Int, length: Int)`{ - UTF8Char* myfrom = recv + my_from*(sizeof(UTF8Char)); + UTF8Char* myfrom = self + my_from*(sizeof(UTF8Char)); UTF8Char* itsfrom = other + its_from*(sizeof(UTF8Char)); memcpy(itsfrom, myfrom, length); `} @@ -196,7 +216,7 @@ redef class FlatString redef fun to_cstring do if real_items != null then return real_items.as(not null) - var new_items = calloc_string(bytelen + 1) + var new_items = new NativeString(bytelen + 1) self.items.copy_to(new_items, bytelen, index[index_from].pos, 0) new_items[bytelen] = '\0' self.real_items = new_items @@ -227,7 +247,7 @@ redef class FlatString redef fun reversed do - var native = calloc_string(self.bytelen + 1) + var native = new NativeString(self.bytelen + 1) var length = self.length var index = self.index var pos = 0 @@ -239,7 +259,7 @@ redef class FlatString var uchar = index[i] var uchar_len = uchar.len ipos -= uchar_len - new_index[pos_index] = new UnicodeChar(ipos, native) + new_index[pos_index] = new UTF8Char(ipos, native) pos_index -= 1 items.copy_to(native, uchar_len, pos, ipos) pos += uchar_len @@ -260,7 +280,7 @@ redef class FlatString var my_real_len = length var my_real_fin_len = my_real_len * i - var target_string = calloc_string((finlen) + 1) + var target_string = new NativeString((finlen) + 1) var my_index = index var new_index = new StringIndex(my_real_fin_len) @@ -282,13 +302,12 @@ redef class FlatString redef fun to_upper do - var outstr = calloc_string(self.bytelen + 1) + var outstr = new NativeString(self.bytelen + 1) var out_index = 0 var index = self.index var ipos = 0 var max = length - var items = self.items while ipos < max do var u = index[ipos].to_upper @@ -304,13 +323,12 @@ redef class FlatString redef fun to_lower do - var outstr = calloc_string(self.bytelen + 1) + var outstr = new NativeString(self.bytelen + 1) var out_index = 0 var index = self.index var ipos = 0 var max = length - var items = self.items while ipos < max do var u = index[ipos].to_lower @@ -324,6 +342,30 @@ redef class FlatString return outstr.to_s_with_length(self.bytelen) end + redef fun output + do + var i = self.index_from + var imax = self.index_to + while i <= imax do + index[i].output + i += 1 + end + end + +end + +redef class FlatBuffer + + # Fix for this particular implementation + # + # Since the to_s of a FlatBuffer now builds using + # the old String contructor, this breaks everything. + # + # This will disappear when UTF8 is fully-supported + redef fun to_s do + written = false + return to_cstring.to_s_with_length(length) + end end redef class NativeString @@ -331,15 +373,15 @@ redef class NativeString # Creates the index for said NativeString # `length` is the size of the CString (in bytes, up to the first \0) # real_len is just a way to store the length (UTF-8 characters) - private fun make_index(length: Int, real_len: Container[Int]): StringIndex import Container[Int].item=, UnicodeChar.len `{ + private fun make_index(length: Int, real_len: Container[Int]): StringIndex import Container[Int].item=, UTF8Char.len `{ int pos = 0; int index_pos = 0; UTF8Char* index = malloc(length*sizeof(UTF8Char)); while(pos < length){ UTF8Char* curr = &index[index_pos]; curr->pos = pos; - curr->ns = recv; - pos += UnicodeChar_len(curr); + curr->ns = self; + pos += UTF8Char_len(curr); index_pos ++; } Container_of_Int_item__assign(real_len, index_pos); @@ -352,7 +394,7 @@ redef class NativeString return to_s_with_length(len) end - redef fun to_s_with_length(len: Int): FlatString + redef fun to_s_with_length(len) do var real_len = new Container[Int](0) var x = make_index(len, real_len) @@ -364,8 +406,22 @@ redef class NativeString var real_len = new Container[Int](0) var length = cstring_length var x = make_index(length, real_len) - var new_self = calloc_string(length + 1) + var new_self = new NativeString(length + 1) copy_to(new_self, length, 0, 0) return new FlatString.with_infos_index(new_self, real_len.item, 0, real_len.item - 1, x, length) end end + +redef class FileWriter + redef fun write(s) + do + assert is_writable + if s isa FlatText then + if s isa FlatString then + write_native(s.to_cstring, s.bytelen) + else + write_native(s.to_cstring, s.length) + end + else for i in s.substrings do write_native(i.to_cstring, i.length) + end +end