lib/core/text/native.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # This file is free software, which comes along with NIT.  This software is
   4 # distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
   5 # without  even  the implied warranty of  MERCHANTABILITY or  FITNESS FOR A
   6 # PARTICULAR PURPOSE.  You can modify it is you want,  provided this header
   7 # is kept unaltered, and a notification of the changes is added.
   8 # You  are  allowed  to  redistribute it and sell it, alone or is a part of
   9 # another product.
  10
  11 # Native structures for text and bytes
  12 module native
  13
  14 import kernel
  15 import math
  16
  17 in "C" `{
  18 #ifdef __linux__
  19         #include <endian.h>
  20 #endif
  21 #ifdef __APPLE__
  22         #include <libkern/OSByteOrder.h>
  23         #define be32toh(x) OSSwapBigToHostInt32(x)
  24 #endif
  25
  26 #ifdef __pnacl__
  27         #define be16toh(val) (((val) >> 8) | ((val) << 8))
  28         #define be32toh(val) ((be16toh((val) << 16) | (be16toh((val) >> 16))))
  29 #endif
  30 #ifndef be32toh
  31         #define be32toh(val) betoh32(val)
  32 #endif
  33 `}
  34
  35 redef class Byte
  36         # Gives the length of the UTF-8 char starting with `self`
  37         fun u8len: Int do
  38                 if self & 0b1000_0000u8 == 0u8 then
  39                         return 1
  40                 else if self & 0b1110_0000u8 == 0b1100_0000u8 then
  41                         return 2
  42                 else if self & 0b1111_0000u8 == 0b1110_0000u8 then
  43                         return 3
  44                 else if self & 0b1111_1000u8 == 0b1111_0000u8 then
  45                         return 4
  46                 else
  47                         return 1
  48                 end
  49         end
  50 end
  51
  52 redef class Int
  53         # Returns the code_point from a utf16 surrogate pair
  54         #
  55         #     assert 0xD83DDE02.from_utf16_surr == 0x1F602
  56         fun from_utf16_surr: Int do
  57                 var hi = (self & 0xFFFF0000) >> 16
  58                 var lo = self & 0xFFFF
  59                 var cp = 0
  60                 cp += (hi - 0xD800) << 10
  61                 cp += lo - 0xDC00
  62                 cp += 0x10000
  63                 return cp
  64         end
  65 end
  66
  67 # Native strings are simple C char *
  68 extern class NativeString `{ char* `}
  69         # Creates a new NativeString with a capacity of `length`
  70         new(length: Int) is intern
  71
  72         # Returns a char* starting at `index`.
  73         #
  74         # WARNING: Unsafe for extern code, use only for temporary
  75         # pointer manipulation purposes (e.g. write to file or such)
  76         fun fast_cstring(index: Int): NativeString is intern
  77
  78         # Get char at `index`.
  79         fun [](index: Int): Byte is intern
  80
  81         # Set char `item` at index.
  82         fun []=(index: Int, item: Byte) is intern
  83
  84         # Copy `self` to `dest`.
  85         fun copy_to(dest: NativeString, length: Int, from: Int, to: Int) is intern
  86
  87         redef fun ==(o) is intern do return is_same_instance(o)
  88
  89         redef fun !=(o) is intern do return not is_same_instance(o)
  90
  91         # Position of the first nul character.
  92         fun cstring_length: Int
  93         do
  94                 var l = 0
  95                 while self[l] != 0u8 do l += 1
  96                 return l
  97         end
  98
  99         # Parse `self` as an Int.
 100         fun atoi: Int is intern
 101
 102         # Parse `self` as a Float.
 103         fun atof: Float `{ return atof(self); `}
 104
 105         # Gets the UTF-8 char at index `pos`
 106         #
 107         # Index is expressed in Unicode chars
 108         #
 109         # ~~~raw
 110         #     assert "かきく".as(FlatString).items.char_at(0) == 'か'
 111         # ~~~
 112         #
 113         # If the char at position pos is an invalid Unicode char,
 114         # the Unicode replacement character � (0xFFFD) will be used.
 115         #
 116         # ~~~raw
 117         #     assert "かきく".as(FlatString).items.char_at(1) == '�'
 118         # ~~~
 119         fun char_at(pos: Int): Char do
 120                 var c = self[pos]
 121                 if c & 0x80u8 == 0u8 then return c.ascii
 122                 var b = fetch_4_hchars(pos)
 123                 var ret = 0
 124                 if b & 0xC00000 != 0x800000 then return 0xFFFD.code_point
 125                 if b & 0xE0000000 == 0xC0000000 then
 126                         ret |= (b & 0x1F000000) >> 18
 127                         ret |= (b & 0x3F0000) >> 16
 128                         return ret.code_point
 129                 end
 130                 if not b & 0xC000 == 0x8000 then return 0xFFFD.code_point
 131                 if b & 0xF0000000 == 0xE0000000 then
 132                         ret |= (b & 0xF000000) >> 12
 133                         ret |= (b & 0x3F0000) >> 10
 134                         ret |= (b & 0x3F00) >> 8
 135                         return ret.code_point
 136                 end
 137                 if not b & 0xC0 == 0x80 then return 0xFFFD.code_point
 138                 if b & 0xF8000000 == 0xF0000000 then
 139                         ret |= (b.to_i & 0x7000000) >> 6
 140                         ret |= (b.to_i & 0x3F0000) >> 4
 141                         ret |= (b.to_i & 0x3F00) >> 2
 142                         ret |= b.to_i & 0x3F
 143                         return ret.code_point
 144                 end
 145                 return 0xFFFD.code_point
 146         end
 147
 148         # Gets the byte index of char at position `n` in UTF-8 String
 149         fun char_to_byte_index(n: Int): Int do return char_to_byte_index_cached(n, 0, 0)
 150
 151         # Gets the length of the character at position `pos` (1 if invalid sequence)
 152         fun length_of_char_at(pos: Int): Int do
 153                 var c = self[pos]
 154                 if c & 0x80u8 == 0x00u8 then
 155                         return 1
 156                 else if c & 0xE0u8 == 0xC0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 then
 157                         return 2
 158                 else if c & 0xF0u8 == 0xE0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 and self[pos + 2] & 0xC0u8 == 0x80u8 then
 159                         return 3
 160                 else if c & 0xF8u8 == 0xF0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 and self[pos + 2] & 0xC0u8 == 0x80u8 and self[pos + 3] & 0xC0u8 == 0x80u8 then
 161                         return 4
 162                 else
 163                         return 1
 164                 end
 165         end
 166
 167         # Gets the byte index of char at position `n` in UTF-8 String
 168         #
 169         # `char_from` and `byte_from` are cached values to seek from.
 170         #
 171         # NOTE: char_from and byte_from are not guaranteed to be valid cache values
 172         # It it up to the client to ensure the validity of the information
 173         fun char_to_byte_index_cached(n, char_from, byte_from: Int): Int do
 174                 var ns_i = byte_from
 175                 var my_i = char_from
 176
 177                 var dist = n - my_i
 178
 179                 while dist > 0 do
 180                         while dist >= 4 do
 181                                 var i = fetch_4_chars(ns_i)
 182                                 if i & 0x80808080 != 0 then break
 183                                 ns_i += 4
 184                                 my_i += 4
 185                                 dist -= 4
 186                         end
 187                         if dist == 0 then break
 188                         ns_i += length_of_char_at(ns_i)
 189                         my_i += 1
 190                         dist -= 1
 191                 end
 192
 193                 while dist < 0 do
 194                         while dist <= -4 do
 195                                 var i = fetch_4_chars(ns_i - 4)
 196                                 if i & 0x80808080 != 0 then break
 197                                 ns_i -= 4
 198                                 my_i -= 4
 199                                 dist += 4
 200                         end
 201                         if dist == 0 then break
 202                         ns_i = find_beginning_of_char_at(ns_i - 1)
 203                         my_i -= 1
 204                         dist += 1
 205                 end
 206
 207                 return ns_i
 208         end
 209
 210         # Gets the char index of byte at position `n` in a UTF-8 String
 211         #
 212         # `char_from` and `byte_from` are cached values to seek from.
 213         #
 214         # NOTE: char_from and byte_from are not guaranteed to be valid cache values
 215         # It it up to the client to ensure the validity of the information
 216         fun byte_to_char_index_cached(n, char_from, byte_from: Int): Int do
 217                 var ns_i = byte_from
 218                 var my_i = char_from
 219
 220                 while ns_i < n do
 221                         ns_i += length_of_char_at(ns_i)
 222                         my_i += 1
 223                 end
 224
 225                 while ns_i > n do
 226                         ns_i = find_beginning_of_char_at(ns_i - 1)
 227                         my_i -= 1
 228                 end
 229
 230                 return my_i
 231         end
 232
 233         # Returns the beginning position of the char at position `pos`
 234         #
 235         # If the char is invalid UTF-8, `pos` is returned as-is
 236         #
 237         # ~~~raw
 238         #       assert "abc".items.find_beginning_of_char_at(2) == 2
 239         #       assert "か".items.find_beginning_of_char_at(1) == 0
 240         #       assert [0x41u8, 233u8].to_s.items.find_beginning_of_char_at(1) == 1
 241         # ~~~
 242         fun find_beginning_of_char_at(pos: Int): Int do
 243                 var endpos = pos
 244                 var c = self[pos]
 245                 if c & 0x80u8 == 0x00u8 then return pos
 246                 while c & 0xC0u8 == 0x80u8 do
 247                         pos -= 1
 248                         c = self[pos]
 249                 end
 250                 var stpos = pos
 251                 if length_of_char_at(stpos) >= (endpos - stpos + 1) then return pos
 252                 return endpos
 253         end
 254
 255         # Number of UTF-8 characters in `self` starting at `from`, for a length of `bytelen`
 256         fun utf8_length(from, bytelen: Int): Int is intern do
 257                 var st = from
 258                 var ln = 0
 259                 while bytelen > 0 do
 260                         while bytelen >= 4 do
 261                                 var i = fetch_4_chars(st)
 262                                 if i & 0x80808080 != 0 then break
 263                                 bytelen -= 4
 264                                 st += 4
 265                                 ln += 4
 266                         end
 267                         if bytelen == 0 then break
 268                         var cln = length_of_char_at(st)
 269                         st += cln
 270                         ln += 1
 271                         bytelen -= cln
 272                 end
 273                 return ln
 274         end
 275
 276         # Fetch 4 chars in `self` at `pos`
 277         fun fetch_4_chars(pos: Int): Int is intern do return fetch_4_ffi(pos)
 278
 279         # Fetch 4 chars in `self` at `pos`
 280         fun fetch_4_hchars(pos: Int): Int is intern do return fetch_4h_ffi(pos)
 281
 282         # FIXME: To remove when bootstrap supports PR #1898
 283         private fun fetch_4_ffi(pos: Int): Int `{ return (long)*((uint32_t*)(self+pos)); `}
 284         private fun fetch_4h_ffi(pos: Int): Int `{ return (long)be32toh(*((uint32_t*)(self+pos))); `}
 285 end