lib/string_experimentations/utf8.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Introduces UTF-8 as internal encoding for Strings in Nit.
  16 module utf8
  17
  18 intrude import standard::string
  19
  20 in "C Header" `{
  21
  22 #include <stdio.h>
  23 #include <string.h>
  24 #include <stdint.h>
  25
  26 typedef struct {
  27         long pos;
  28         char* ns;
  29 } UTF8Char;
  30
  31 `}
  32
  33 # UTF-8 char as defined in RFC-3629, e.g. 1-4 Bytes
  34 #
  35 # A UTF-8 char has its bytes stored in a NativeString (char*)
  36 extern class UnicodeChar `{ UTF8Char* `}
  37
  38         new(pos: Int, ns: NativeString) `{
  39                 UTF8Char* u = malloc(sizeof(UTF8Char));
  40                 u->pos = pos;
  41                 u->ns = ns;
  42                 return u;
  43         `}
  44
  45         # Real length of the char in UTF8
  46         #
  47         # As per the specification :
  48         #
  49         #  Length  |        UTF-8 octet sequence
  50         #          |              (binary)
  51         # ---------+-------------------------------------------------
  52         #  1       | 0xxxxxxx
  53         #  2       | 110xxxxx 10xxxxxx
  54         #  3       | 1110xxxx 10xxxxxx 10xxxxxx
  55         #  4       | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  56         private fun len: Int `{
  57                 char* ns = recv->ns;
  58                 int pos = recv->pos;
  59                 char nspos = ns[pos];
  60                 if((nspos & 0x80) == 0x00){ return 1;}
  61                 if((nspos & 0xE0) == 0xC0){ return 2;}
  62                 if((nspos & 0xF0) == 0xE0){ return 3;}
  63                 if((nspos & 0xF7) == 0xF0){ return 4;}
  64                 // Invalid character
  65                 return 1;
  66         `}
  67
  68         # Position in containing NativeString
  69         private fun pos: Int `{
  70                 return recv->pos;
  71         `}
  72
  73         private fun pos=(p: Int) `{recv->pos = p;`}
  74
  75         # C char* wrapping the char
  76         fun ns: NativeString `{
  77                 return recv->ns;
  78         `}
  79
  80         redef fun to_s import NativeString.to_s_with_length `{
  81                 int len = utf8___UnicodeChar_len___impl(recv);
  82                 char* r = malloc(len + 1);
  83                 r[len] = '\0';
  84                 char* src = (recv->ns + recv->pos);
  85                 memcpy(r, src, len);
  86                 return NativeString_to_s_with_length(r, len);
  87         `}
  88 end
  89
  90 # A `StringIndex` is used to keep track of the position of characters in a `FlatString` object
  91 #
  92 # It becomes mandatory for UTF-8 strings since characters do not have a fixed size.
  93 private extern class StringIndex `{ UTF8Char* `}
  94
  95         new(size: Int) `{ return malloc(size*sizeof(UTF8Char)); `}
  96
  97         # Sets the character at `index` as `item`
  98         fun []=(index: Int, item: UnicodeChar) `{ recv[index] = *item; `}
  99
 100         # Gets the character at position `id`
 101         fun [](id: Int): UnicodeChar `{ return &recv[id]; `}
 102
 103         # Copies a part of self starting at index `my_from` of length `length` into `other`, starting at `its_from`
 104         fun copy_to(other: StringIndex, my_from: Int, its_from: Int, length: Int)`{
 105                 UTF8Char* myfrom = recv + my_from*(sizeof(UTF8Char));
 106                 UTF8Char* itsfrom = other + its_from*(sizeof(UTF8Char));
 107                 memcpy(itsfrom, myfrom, length);
 108         `}
 109 end
 110
 111 redef class FlatString
 112
 113         # Index of the characters of the FlatString
 114         private var index: StringIndex
 115
 116         # Length in bytes of the string (e.g. the length of the C string)
 117         var bytelen: Int
 118
 119         private init with_infos_index(items: NativeString, len: Int, index_from: Int, index_to: Int, index: StringIndex, bytelen: Int)
 120         do
 121                 self.items = items
 122                 length = len
 123                 self.index_from = index_from
 124                 self.index_to = index_to
 125                 self.index = index
 126                 self.bytelen = bytelen
 127         end
 128
 129 end
 130
 131 redef class NativeString
 132
 133         # Creates the index for said NativeString
 134         # `length` is the size of the CString (in bytes, up to the first \0)
 135         # real_len is just a way to store the length (UTF-8 characters)
 136         private fun make_index(length: Int, real_len: Container[Int]): StringIndex import Container[Int].item=, UnicodeChar.len `{
 137                 int pos = 0;
 138                 int index_pos = 0;
 139                 UTF8Char* index = malloc(length*sizeof(UTF8Char));
 140                 while(pos < length){
 141                         UTF8Char* curr = &index[index_pos];
 142                         curr->pos = pos;
 143                         curr->ns = recv;
 144                         pos += UnicodeChar_len(curr);
 145                         index_pos ++;
 146                 }
 147                 Container_of_Int_item__assign(real_len, index_pos);
 148                 return index;
 149         `}
 150
 151         redef fun to_s: FlatString
 152         do
 153                 var len = cstring_length
 154                 return to_s_with_length(len)
 155         end
 156
 157         redef fun to_s_with_length(len: Int): FlatString
 158         do
 159                 var real_len = new Container[Int](0)
 160                 var x = make_index(len, real_len)
 161                 return new FlatString.with_infos_index(self, real_len.item, 0, real_len.item - 1, x, len)
 162         end
 163
 164         redef fun to_s_with_copy
 165         do
 166                 var real_len = new Container[Int](0)
 167                 var length = cstring_length
 168                 var x = make_index(length, real_len)
 169                 var new_self = calloc_string(length + 1)
 170                 copy_to(new_self, length, 0, 0)
 171                 return new FlatString.with_infos_index(new_self, real_len.item, 0, real_len.item - 1, x, length)
 172         end
 173 end