lib/core/text/native.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # This file is free software, which comes along with NIT.  This software is
   4 # distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
   5 # without  even  the implied warranty of  MERCHANTABILITY or  FITNESS FOR A
   6 # PARTICULAR PURPOSE.  You can modify it is you want,  provided this header
   7 # is kept unaltered, and a notification of the changes is added.
   8 # You  are  allowed  to  redistribute it and sell it, alone or is a part of
   9 # another product.
  10
  11 # Native structures for text and bytes
  12 module native
  13
  14 import kernel
  15 import math
  16
  17 redef class Byte
  18         # Gives the length of the UTF-8 char starting with `self`
  19         private fun u8len: Int do
  20                 if self & 0b1000_0000u8 == 0u8 then
  21                         return 1
  22                 else if self & 0b1110_0000u8 == 0b1100_0000u8 then
  23                         return 2
  24                 else if self & 0b1111_0000u8 == 0b1110_0000u8 then
  25                         return 3
  26                 else if self & 0b1111_1000u8 == 0b1111_0000u8 then
  27                         return 4
  28                 else
  29                         return 1
  30                 end
  31         end
  32 end
  33
  34 # Native strings are simple C char *
  35 extern class NativeString `{ char* `}
  36         # Creates a new NativeString with a capacity of `length`
  37         new(length: Int) is intern
  38
  39         # Returns a char* starting at `index`.
  40         #
  41         # WARNING: Unsafe for extern code, use only for temporary
  42         # pointer manipulation purposes (e.g. write to file or such)
  43         fun fast_cstring(index: Int): NativeString is intern
  44
  45         # Get char at `index`.
  46         fun [](index: Int): Byte is intern
  47
  48         # Set char `item` at index.
  49         fun []=(index: Int, item: Byte) is intern
  50
  51         # Copy `self` to `dest`.
  52         fun copy_to(dest: NativeString, length: Int, from: Int, to: Int) is intern
  53
  54         # Position of the first nul character.
  55         fun cstring_length: Int
  56         do
  57                 var l = 0
  58                 while self[l] != 0u8 do l += 1
  59                 return l
  60         end
  61
  62         # Parse `self` as an Int.
  63         fun atoi: Int is intern
  64
  65         # Parse `self` as a Float.
  66         fun atof: Float `{ return atof(self); `}
  67
  68         # Gets the UTF-8 char at index `pos`
  69         #
  70         # Index is expressed in Unicode chars
  71         #
  72         # ~~~raw
  73         #     assert "かきく".as(FlatString).items.char_at(0) == 'か'
  74         # ~~~
  75         #
  76         # If the char at position pos is an invalid Unicode char,
  77         # the Unicode replacement character � (0xFFFD) will be used.
  78         #
  79         # ~~~raw
  80         #     assert "かきく".as(FlatString).items.char_at(1) == '�'
  81         # ~~~
  82         fun char_at(pos: Int): Char `{
  83                 char c = self[pos];
  84                 if((c & 0x80) == 0x00) return (uint32_t)c;
  85                 if(((c & 0xE0) == 0xC0) && ((self[pos + 1] & 0xC0) == 0x80)) return ((((uint32_t)c) & 0x1F) << 6) + ((((uint32_t)self[pos + 1] & 0x3F)));
  86                 if(((c & 0xF0) == 0xE0) && ((self[pos + 1] & 0xC0) == 0x80) && ((self[pos + 2] & 0xC0) == 0x80)) return ((((uint32_t)c) & 0xF) << 12) + ((((uint32_t)self[pos + 1]) & 0x3F) << 6) + ((((uint32_t)self[pos + 2] & 0x3F)));
  87                 if(((c & 0xF8) == 0xF0) && ((self[pos + 1] & 0xC0) == 0x80) && ((self[pos + 2] & 0xC0) == 0x80) && ((self[pos + 3] & 0xC0) == 0x80)) return ((((uint32_t)c) & 0x7) << 18) + ((((uint32_t)self[pos + 1]) & 0x3F) << 12) + ((((uint32_t)self[pos + 2]) & 0x3F) << 6) + ((((uint32_t)self[pos + 3] & 0x3F)));
  88                 return 0xFFFD;
  89         `}
  90
  91         # Gets the byte index of char at position `n` in UTF-8 String
  92         fun char_to_byte_index(n: Int): Int do return char_to_byte_index_cached(n, 0, 0)
  93
  94         # Gets the length of the character at position `pos` (1 if invalid sequence)
  95         fun length_of_char_at(pos: Int): Int do
  96                 var c = self[pos]
  97                 if c & 0x80u8 == 0x00u8 then
  98                         return 1
  99                 else if c & 0xE0u8 == 0xC0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 then
 100                         return 2
 101                 else if c & 0xF0u8 == 0xE0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 and self[pos + 2] & 0xC0u8 == 0x80u8 then
 102                         return 3
 103                 else if c & 0xF8u8 == 0xF0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 and self[pos + 2] & 0xC0u8 == 0x80u8 and self[pos + 3] & 0xC0u8 == 0x80u8 then
 104                         return 4
 105                 else
 106                         return 1
 107                 end
 108         end
 109
 110         # Gets the byte index of char at position `n` in UTF-8 String
 111         #
 112         # `char_from` and `byte_from` are cached values to seek from.
 113         #
 114         # NOTE: char_from and byte_from are not guaranteed to be valid cache values
 115         # It it up to the client to ensure the validity of the information
 116         fun char_to_byte_index_cached(n, char_from, byte_from: Int): Int do
 117                 var ns_i = byte_from
 118                 var my_i = char_from
 119
 120                 while my_i < n do
 121                         ns_i += length_of_char_at(ns_i)
 122                         my_i += 1
 123                 end
 124
 125                 while my_i > n do
 126                         ns_i = find_beginning_of_char_at(ns_i - 1)
 127                         my_i -= 1
 128                 end
 129
 130                 return ns_i
 131         end
 132
 133         # Gets the char index of byte at position `n` in a UTF-8 String
 134         #
 135         # `char_from` and `byte_from` are cached values to seek from.
 136         #
 137         # NOTE: char_from and byte_from are not guaranteed to be valid cache values
 138         # It it up to the client to ensure the validity of the information
 139         fun byte_to_char_index_cached(n, char_from, byte_from: Int): Int do
 140                 var ns_i = byte_from
 141                 var my_i = char_from
 142
 143                 while ns_i < n do
 144                         ns_i += length_of_char_at(ns_i)
 145                         my_i += 1
 146                 end
 147
 148                 while ns_i > n do
 149                         ns_i = find_beginning_of_char_at(ns_i - 1)
 150                         my_i -= 1
 151                 end
 152
 153                 return my_i
 154         end
 155
 156         # Returns the beginning position of the char at position `pos`
 157         #
 158         # If the char is invalid UTF-8, `pos` is returned as-is
 159         #
 160         # ~~~raw
 161         #       assert "abc".items.find_beginning_of_char_at(2) == 2
 162         #       assert "か".items.find_beginning_of_char_at(1) == 0
 163         #       assert [0x41u8, 233u8].to_s.items.find_beginning_of_char_at(1) == 1
 164         # ~~~
 165         fun find_beginning_of_char_at(pos: Int): Int do
 166                 var endpos = pos
 167                 var c = self[pos]
 168                 while c & 0xC0u8 == 0x80u8 do
 169                         pos -= 1
 170                         c = self[pos]
 171                 end
 172                 var stpos = pos
 173                 if length_of_char_at(stpos) >= (endpos - stpos + 1) then return pos
 174                 return endpos
 175         end
 176
 177         # Number of UTF-8 characters in `self` between positions `from` and `to`
 178         fun utf8_length(from, to: Int): Int do
 179                 var st = from
 180                 var lst = to
 181                 var ln = 0
 182                 while st <= lst do
 183                         st += length_of_char_at(st)
 184                         ln += 1
 185                 end
 186                 return ln
 187         end
 188 end