lib/string_experimentations/utf8.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Introduces UTF-8 as internal encoding for Strings in Nit.
  16 module utf8
  17
  18 intrude import standard::string
  19
  20 in "C Header" `{
  21
  22 #include <stdio.h>
  23 #include <string.h>
  24 #include <stdint.h>
  25
  26 typedef struct {
  27         long pos;
  28         char* ns;
  29 } UTF8Char;
  30
  31 `}
  32
  33 # UTF-8 char as defined in RFC-3629, e.g. 1-4 Bytes
  34 #
  35 # A UTF-8 char has its bytes stored in a NativeString (char*)
  36 extern class UnicodeChar `{ UTF8Char* `}
  37
  38         new(pos: Int, ns: NativeString) `{
  39                 UTF8Char* u = malloc(sizeof(UTF8Char));
  40                 u->pos = pos;
  41                 u->ns = ns;
  42                 return u;
  43         `}
  44
  45         # Real length of the char in UTF8
  46         #
  47         # As per the specification :
  48         #
  49         #  Length  |        UTF-8 octet sequence
  50         #          |              (binary)
  51         # ---------+-------------------------------------------------
  52         #  1       | 0xxxxxxx
  53         #  2       | 110xxxxx 10xxxxxx
  54         #  3       | 1110xxxx 10xxxxxx 10xxxxxx
  55         #  4       | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  56         private fun len: Int `{
  57                 char* ns = recv->ns;
  58                 int pos = recv->pos;
  59                 char nspos = ns[pos];
  60                 if((nspos & 0x80) == 0x00){ return 1;}
  61                 if((nspos & 0xE0) == 0xC0){ return 2;}
  62                 if((nspos & 0xF0) == 0xE0){ return 3;}
  63                 if((nspos & 0xF7) == 0xF0){ return 4;}
  64                 // Invalid character
  65                 return 1;
  66         `}
  67
  68         # Position in containing NativeString
  69         private fun pos: Int `{
  70                 return recv->pos;
  71         `}
  72
  73         private fun pos=(p: Int) `{recv->pos = p;`}
  74
  75         # C char* wrapping the char
  76         fun ns: NativeString `{
  77                 return recv->ns;
  78         `}
  79
  80         # Returns the Unicode code point representing the character
  81         #
  82         # Note : A unicode character might not be a visible glyph, but it will be used to determine canonical equivalence
  83         fun code_point: Int import UnicodeChar.len `{
  84                 switch(UnicodeChar_len(recv)){
  85                         case 1:
  86                                 return (long)(0x7F & (unsigned char)recv->ns[recv->pos]);
  87                         case 2:
  88                                 return 0 | ((0x1F & (unsigned char)recv->ns[recv->pos]) << 6) | (0x3F & (unsigned char)recv->ns[recv->pos+1]);
  89                         case 3:
  90                                 return 0 | ((0x0F & (unsigned char)recv->ns[recv->pos]) << 12) |
  91                                 ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 6) |
  92                                 (0x3F & (unsigned char)recv->ns[recv->pos+2]);
  93                         case 4:
  94                                 return 0 | ((0x07 & (unsigned char)recv->ns[recv->pos]) << 18) |
  95                                 ((0x3F & (unsigned char)recv->ns[recv->pos+1]) << 12) |
  96                                 ((0x3F & (unsigned char)recv->ns[recv->pos+2]) << 6) |
  97                                 (0x3F & (unsigned char)recv->ns[recv->pos+3]);
  98                 }
  99         `}
 100
 101         redef fun to_s import NativeString.to_s_with_length `{
 102                 int len = utf8___UnicodeChar_len___impl(recv);
 103                 char* r = malloc(len + 1);
 104                 r[len] = '\0';
 105                 char* src = (recv->ns + recv->pos);
 106                 memcpy(r, src, len);
 107                 return NativeString_to_s_with_length(r, len);
 108         `}
 109 end
 110
 111 # A `StringIndex` is used to keep track of the position of characters in a `FlatString` object
 112 #
 113 # It becomes mandatory for UTF-8 strings since characters do not have a fixed size.
 114 private extern class StringIndex `{ UTF8Char* `}
 115
 116         new(size: Int) `{ return malloc(size*sizeof(UTF8Char)); `}
 117
 118         # Sets the character at `index` as `item`
 119         fun []=(index: Int, item: UnicodeChar) `{ recv[index] = *item; `}
 120
 121         # Gets the character at position `id`
 122         fun [](id: Int): UnicodeChar `{ return &recv[id]; `}
 123
 124         # Copies a part of self starting at index `my_from` of length `length` into `other`, starting at `its_from`
 125         fun copy_to(other: StringIndex, my_from: Int, its_from: Int, length: Int)`{
 126                 UTF8Char* myfrom = recv + my_from*(sizeof(UTF8Char));
 127                 UTF8Char* itsfrom = other + its_from*(sizeof(UTF8Char));
 128                 memcpy(itsfrom, myfrom, length);
 129         `}
 130 end
 131
 132 redef class FlatString
 133
 134         # Index of the characters of the FlatString
 135         private var index: StringIndex
 136
 137         # Length in bytes of the string (e.g. the length of the C string)
 138         var bytelen: Int
 139
 140         private init with_infos_index(items: NativeString, len: Int, index_from: Int, index_to: Int, index: StringIndex, bytelen: Int)
 141         do
 142                 self.items = items
 143                 length = len
 144                 self.index_from = index_from
 145                 self.index_to = index_to
 146                 self.index = index
 147                 self.bytelen = bytelen
 148         end
 149
 150         redef fun *(i)
 151         do
 152                 assert i >= 0
 153
 154                 var mylen = self.bytelen
 155                 var finlen = mylen * i
 156
 157                 var my_items = self.items
 158
 159                 var my_real_len = length
 160                 var my_real_fin_len = my_real_len * i
 161
 162                 var target_string = calloc_string((finlen) + 1)
 163
 164                 var my_index = index
 165                 var new_index = new StringIndex(my_real_fin_len)
 166
 167                 target_string[finlen] = '\0'
 168
 169                 var current_last = 0
 170                 var curr_index = 0
 171
 172                 for iteration in [1 .. i] do
 173                         my_items.copy_to(target_string, mylen, index_from, current_last)
 174                         my_index.copy_to(new_index, length, 0, curr_index)
 175                         current_last += mylen
 176                 end
 177
 178                 return new FlatString.with_infos_index(target_string, my_real_fin_len, 0, my_real_fin_len -1, new_index, finlen)
 179
 180         end
 181
 182 end
 183
 184 redef class NativeString
 185
 186         # Creates the index for said NativeString
 187         # `length` is the size of the CString (in bytes, up to the first \0)
 188         # real_len is just a way to store the length (UTF-8 characters)
 189         private fun make_index(length: Int, real_len: Container[Int]): StringIndex import Container[Int].item=, UnicodeChar.len `{
 190                 int pos = 0;
 191                 int index_pos = 0;
 192                 UTF8Char* index = malloc(length*sizeof(UTF8Char));
 193                 while(pos < length){
 194                         UTF8Char* curr = &index[index_pos];
 195                         curr->pos = pos;
 196                         curr->ns = recv;
 197                         pos += UnicodeChar_len(curr);
 198                         index_pos ++;
 199                 }
 200                 Container_of_Int_item__assign(real_len, index_pos);
 201                 return index;
 202         `}
 203
 204         redef fun to_s: FlatString
 205         do
 206                 var len = cstring_length
 207                 return to_s_with_length(len)
 208         end
 209
 210         redef fun to_s_with_length(len: Int): FlatString
 211         do
 212                 var real_len = new Container[Int](0)
 213                 var x = make_index(len, real_len)
 214                 return new FlatString.with_infos_index(self, real_len.item, 0, real_len.item - 1, x, len)
 215         end
 216
 217         redef fun to_s_with_copy
 218         do
 219                 var real_len = new Container[Int](0)
 220                 var length = cstring_length
 221                 var x = make_index(length, real_len)
 222                 var new_self = calloc_string(length + 1)
 223                 copy_to(new_self, length, 0, 0)
 224                 return new FlatString.with_infos_index(new_self, real_len.item, 0, real_len.item - 1, x, length)
 225         end
 226 end