lib/string_experimentations/utf8.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Introduces UTF-8 as internal encoding for Strings in Nit.
  16 module utf8
  17
  18 intrude import standard::string
  19 intrude import standard::file
  20
  21 in "C Header" `{
  22
  23 #include <stdio.h>
  24 #include <string.h>
  25 #include <stdint.h>
  26
  27 typedef struct {
  28         long pos;
  29         char* ns;
  30 } UTF8Char;
  31
  32 `}
  33
  34 # UTF-8 char as defined in RFC-3629, e.g. 1-4 Bytes
  35 #
  36 # A UTF-8 char has its bytes stored in a NativeString (char*)
  37 extern class UnicodeChar `{ UTF8Char* `}
  38
  39         new(pos: Int, ns: NativeString) `{
  40                 UTF8Char* u = malloc(sizeof(UTF8Char));
  41                 u->pos = pos;
  42                 u->ns = ns;
  43                 return u;
  44         `}
  45
  46         # Real length of the char in UTF8
  47         #
  48         # As per the specification :
  49         #
  50         #  Length  |        UTF-8 octet sequence
  51         #          |              (binary)
  52         # ---------+-------------------------------------------------
  53         #  1       | 0xxxxxxx
  54         #  2       | 110xxxxx 10xxxxxx
  55         #  3       | 1110xxxx 10xxxxxx 10xxxxxx
  56         #  4       | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  57         private fun len: Int `{
  58                 char* ns = recv->ns;
  59                 int pos = recv->pos;
  60                 char nspos = ns[pos];
  61                 if((nspos & 0x80) == 0x00){ return 1;}
  62                 if((nspos & 0xE0) == 0xC0){ return 2;}
  63                 if((nspos & 0xF0) == 0xE0){ return 3;}
  64                 if((nspos & 0xF7) == 0xF0){ return 4;}
  65                 // Invalid character
  66                 return 1;
  67         `}
  68
  69         # Position in containing NativeString
  70         private fun pos: Int `{
  71                 return recv->pos;
  72         `}
  73
  74         private fun pos=(p: Int) `{recv->pos = p;`}
  75
  76         # C char* wrapping the char
  77         fun ns: NativeString `{
  78                 return recv->ns;
  79         `}
  80
  81         # Returns the Unicode code point representing the character
  82         #
  83         # Note : A unicode character might not be a visible glyph, but it will be used to determine canonical equivalence
  84         fun code_point: Int import UnicodeChar.len `{
  85                 switch(UnicodeChar_len(recv)){
  86                         case 1:
  87                                 return (long)(0x7F & (unsigned char)recv->ns[recv->pos]);
  88                         case 2:
  89                                 return 0 | ((0x1F & (unsigned char)recv->ns[recv->pos]) << 6) | (0x3F & (unsigned char)recv->ns[recv->pos+1]);
  90                         case 3:
  91                                 return 0 | ((0x0F & (unsigned char)recv->ns[recv->pos]) << 12) |
  92                                 ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 6) |
  93                                 (0x3F & (unsigned char)recv->ns[recv->pos+2]);
  94                         case 4:
  95                                 return 0 | ((0x07 & (unsigned char)recv->ns[recv->pos]) << 18) |
  96                                 ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 12) |
  97                                 ((0x3F & (unsigned char)recv->ns[recv->pos+2]) << 6) |
  98                                 (0x3F & (unsigned char)recv->ns[recv->pos+3]);
  99                 }
 100         `}
 101
 102         # Returns an upper-case version of self
 103         #
 104         # NOTE : Works only on ASCII chars
 105         # TODO : Support unicode for to_upper
 106         fun to_upper: UnicodeChar import UnicodeChar.code_point `{
 107                 int cp = UnicodeChar_code_point(recv);
 108                 if(cp < 97 || cp > 122){ return recv; }
 109                 char* ns = malloc(2);
 110                 ns[1] = '\0';
 111                 char c = recv->ns[recv->pos];
 112                 ns[0] = c - 32;
 113                 UTF8Char* ret = malloc(sizeof(UTF8Char));
 114                 ret->ns = ns;
 115                 ret->pos = 0;
 116                 return ret;
 117         `}
 118
 119         # Returns an lower-case version of self
 120         #
 121         # NOTE : Works only on ASCII chars
 122         # TODO : Support unicode for to_upper
 123         fun to_lower: UnicodeChar import UnicodeChar.code_point `{
 124                 int cp = UnicodeChar_code_point(recv);
 125                 if(cp < 65 || cp > 90){ return recv; }
 126                 char* ns = malloc(2);
 127                 ns[1] = '\0';
 128                 char c = recv->ns[recv->pos];
 129                 ns[0] = c + 32;
 130                 UTF8Char* ret = malloc(sizeof(UTF8Char));
 131                 ret->ns = ns;
 132                 ret->pos = 0;
 133                 return ret;
 134         `}
 135
 136         redef fun ==(o)
 137         do
 138                 if o isa Char then
 139                         if len != 1 then return false
 140                         if code_point == o.ascii then return true
 141                 else if o isa UnicodeChar then
 142                         if len != o.len then return false
 143                         if code_point == o.code_point then return true
 144                 end
 145                 return false
 146         end
 147
 148         redef fun output import UnicodeChar.code_point `{
 149                 switch(UnicodeChar_len(recv)){
 150                         case 1:
 151                                 printf("%c", recv->ns[recv->pos]);
 152                                 break;
 153                         case 2:
 154                                 printf("%c%c", recv->ns[recv->pos], recv->ns[recv->pos + 1]);
 155                                 break;
 156                         case 3:
 157                                 printf("%c%c%c", recv->ns[recv->pos], recv->ns[recv->pos + 1], recv->ns[recv->pos + 2]);
 158                                 break;
 159                         case 4:
 160                                 printf("%c%c%c%c", recv->ns[recv->pos], recv->ns[recv->pos + 1], recv->ns[recv->pos + 2], recv->ns[recv->pos + 3]);
 161                                 break;
 162                 }
 163         `}
 164
 165         redef fun to_s import NativeString.to_s_with_length `{
 166                 int len = utf8___UnicodeChar_len___impl(recv);
 167                 char* r = malloc(len + 1);
 168                 r[len] = '\0';
 169                 char* src = (recv->ns + recv->pos);
 170                 memcpy(r, src, len);
 171                 return NativeString_to_s_with_length(r, len);
 172         `}
 173 end
 174
 175 # A `StringIndex` is used to keep track of the position of characters in a `FlatString` object
 176 #
 177 # It becomes mandatory for UTF-8 strings since characters do not have a fixed size.
 178 private extern class StringIndex `{ UTF8Char* `}
 179
 180         new(size: Int) `{ return malloc(size*sizeof(UTF8Char)); `}
 181
 182         # Sets the character at `index` as `item`
 183         fun []=(index: Int, item: UnicodeChar) `{ recv[index] = *item; `}
 184
 185         # Gets the character at position `id`
 186         fun [](id: Int): UnicodeChar `{ return &recv[id]; `}
 187
 188         # Copies a part of self starting at index `my_from` of length `length` into `other`, starting at `its_from`
 189         fun copy_to(other: StringIndex, my_from: Int, its_from: Int, length: Int)`{
 190                 UTF8Char* myfrom = recv + my_from*(sizeof(UTF8Char));
 191                 UTF8Char* itsfrom = other + its_from*(sizeof(UTF8Char));
 192                 memcpy(itsfrom, myfrom, length);
 193         `}
 194 end
 195
 196 redef class FlatString
 197
 198         # Index of the characters of the FlatString
 199         private var index: StringIndex
 200
 201         # Length in bytes of the string (e.g. the length of the C string)
 202         var bytelen: Int
 203
 204         private init with_infos_index(items: NativeString, len: Int, index_from: Int, index_to: Int, index: StringIndex, bytelen: Int)
 205         do
 206                 self.items = items
 207                 length = len
 208                 self.index_from = index_from
 209                 self.index_to = index_to
 210                 self.index = index
 211                 self.bytelen = bytelen
 212         end
 213
 214         redef fun to_cstring
 215         do
 216                 if real_items != null then return real_items.as(not null)
 217                 var new_items = new NativeString(bytelen + 1)
 218                 self.items.copy_to(new_items, bytelen, index[index_from].pos, 0)
 219                 new_items[bytelen] = '\0'
 220                 self.real_items = new_items
 221                 return new_items
 222         end
 223
 224         redef fun substring(from, count)
 225         do
 226                 assert count >= 0
 227
 228                 if from < 0 then
 229                         count += from
 230                         if count < 0 then count = 0
 231                         from = 0
 232                 end
 233
 234                 if count == 0 then return empty
 235
 236                 var real_from = index_from + from
 237                 var real_to = real_from + count - 1
 238
 239                 if real_to > index_to then real_to = index_to
 240
 241                 var sub_bytelen = (index[real_to].pos - index[from].pos) + index[from].len
 242
 243                 return new FlatString.with_infos_index(items, count, real_from, real_to, index, sub_bytelen)
 244         end
 245
 246         redef fun reversed
 247         do
 248                 var native = new NativeString(self.bytelen + 1)
 249                 var length = self.length
 250                 var index = self.index
 251                 var pos = 0
 252                 var i = 0
 253                 var ipos = bytelen
 254                 var new_index = new StringIndex(length)
 255                 var pos_index = length
 256                 while i < length do
 257                         var uchar = index[i]
 258                         var uchar_len = uchar.len
 259                         ipos -= uchar_len
 260                         new_index[pos_index] = new UnicodeChar(ipos, native)
 261                         pos_index -= 1
 262                         items.copy_to(native, uchar_len, pos, ipos)
 263                         pos += uchar_len
 264                         i += 1
 265                 end
 266                 return new FlatString.with_infos_index(native, length, 0, length-1, new_index, bytelen)
 267         end
 268
 269         redef fun *(i)
 270         do
 271                 assert i >= 0
 272
 273                 var mylen = self.bytelen
 274                 var finlen = mylen * i
 275
 276                 var my_items = self.items
 277
 278                 var my_real_len = length
 279                 var my_real_fin_len = my_real_len * i
 280
 281                 var target_string = new NativeString((finlen) + 1)
 282
 283                 var my_index = index
 284                 var new_index = new StringIndex(my_real_fin_len)
 285
 286                 target_string[finlen] = '\0'
 287
 288                 var current_last = 0
 289                 var curr_index = 0
 290
 291                 for iteration in [1 .. i] do
 292                         my_items.copy_to(target_string, mylen, index_from, current_last)
 293                         my_index.copy_to(new_index, length, 0, curr_index)
 294                         current_last += mylen
 295                 end
 296
 297                 return new FlatString.with_infos_index(target_string, my_real_fin_len, 0, my_real_fin_len -1, new_index, finlen)
 298
 299         end
 300
 301         redef fun to_upper
 302         do
 303                 var outstr = new NativeString(self.bytelen + 1)
 304
 305                 var out_index = 0
 306                 var index = self.index
 307                 var ipos = 0
 308                 var max = length
 309                 var items = self.items
 310
 311                 while ipos < max do
 312                         var u = index[ipos].to_upper
 313                         u.ns.copy_to(outstr, u.len, u.pos, out_index)
 314                         out_index += u.len
 315                         ipos += 1
 316                 end
 317
 318                 outstr[self.bytelen] = '\0'
 319
 320                 return outstr.to_s_with_length(self.bytelen)
 321         end
 322
 323         redef fun to_lower
 324         do
 325                 var outstr = new NativeString(self.bytelen + 1)
 326
 327                 var out_index = 0
 328                 var index = self.index
 329                 var ipos = 0
 330                 var max = length
 331                 var items = self.items
 332
 333                 while ipos < max do
 334                         var u = index[ipos].to_lower
 335                         u.ns.copy_to(outstr, u.len, u.pos, out_index)
 336                         out_index += u.len
 337                         ipos += 1
 338                 end
 339
 340                 outstr[self.bytelen] = '\0'
 341
 342                 return outstr.to_s_with_length(self.bytelen)
 343         end
 344
 345         redef fun output
 346         do
 347                 var i = self.index_from
 348                 var imax = self.index_to
 349                 while i <= imax do
 350                         index[i].output
 351                         i += 1
 352                 end
 353         end
 354
 355 end
 356
 357 redef class FlatBuffer
 358
 359         # Fix for this particular implementation
 360         #
 361         # Since the to_s of a FlatBuffer now builds using
 362         # the old String contructor, this breaks everything.
 363         #
 364         # This will disappear when UTF8 is fully-supported
 365         redef fun to_s do
 366                 written = false
 367                 return to_cstring.to_s_with_length(length)
 368         end
 369 end
 370
 371 redef class NativeString
 372
 373         # Creates the index for said NativeString
 374         # `length` is the size of the CString (in bytes, up to the first \0)
 375         # real_len is just a way to store the length (UTF-8 characters)
 376         private fun make_index(length: Int, real_len: Container[Int]): StringIndex import Container[Int].item=, UnicodeChar.len `{
 377                 int pos = 0;
 378                 int index_pos = 0;
 379                 UTF8Char* index = malloc(length*sizeof(UTF8Char));
 380                 while(pos < length){
 381                         UTF8Char* curr = &index[index_pos];
 382                         curr->pos = pos;
 383                         curr->ns = recv;
 384                         pos += UnicodeChar_len(curr);
 385                         index_pos ++;
 386                 }
 387                 Container_of_Int_item__assign(real_len, index_pos);
 388                 return index;
 389         `}
 390
 391         redef fun to_s: FlatString
 392         do
 393                 var len = cstring_length
 394                 return to_s_with_length(len)
 395         end
 396
 397         redef fun to_s_with_length(len: Int): FlatString
 398         do
 399                 var real_len = new Container[Int](0)
 400                 var x = make_index(len, real_len)
 401                 return new FlatString.with_infos_index(self, real_len.item, 0, real_len.item - 1, x, len)
 402         end
 403
 404         redef fun to_s_with_copy
 405         do
 406                 var real_len = new Container[Int](0)
 407                 var length = cstring_length
 408                 var x = make_index(length, real_len)
 409                 var new_self = new NativeString(length + 1)
 410                 copy_to(new_self, length, 0, 0)
 411                 return new FlatString.with_infos_index(new_self, real_len.item, 0, real_len.item - 1, x, length)
 412         end
 413 end
 414
 415 redef class OFStream
 416         redef fun write(s)
 417         do
 418                 assert is_writable
 419                 if s isa FlatText then
 420                         if s isa FlatString then
 421                                 write_native(s.to_cstring, s.bytelen)
 422                         else
 423                                 write_native(s.to_cstring, s.length)
 424                         end
 425                 else for i in s.substrings do write_native(i.to_cstring, i.length)
 426         end
 427 end