1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # This file is free software, which comes along with NIT. This software is
4 # distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
5 # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
6 # PARTICULAR PURPOSE. You can modify it is you want, provided this header
7 # is kept unaltered, and a notification of the changes is added.
8 # You are allowed to redistribute it and sell it, alone or is a part of
11 # Native structures for text and bytes
22 #include <libkern/OSByteOrder.h>
23 #define be32toh(x) OSSwapBigToHostInt32(x)
26 #define be32toh(val) _byteswap_ulong(val)
30 #define be16toh(val) (((val) >> 8) | ((val) << 8))
31 #define be32toh(val) ((be16toh((val) << 16) | (be16toh((val) >> 16))))
34 #define be32toh(val) betoh32(val)
39 # Gives the length of the UTF-8 char starting with `self`
41 if self & 0b1000_0000u
8 == 0u8
then
43 else if self & 0b1110_0000u
8 == 0b1100_0000u
8 then
45 else if self & 0b1111_0000u
8 == 0b1110_0000u
8 then
47 else if self & 0b1111_1000u
8 == 0b1111_0000u
8 then
54 # Is `self` a valid UTF-8 sequence start ?
57 # assert 0u8.is_valid_utf8_start
58 # assert 0xC0u8.is_valid_utf8_start
59 # assert 0xE0u8.is_valid_utf8_start
60 # assert 0xF0u8.is_valid_utf8_start
62 fun is_valid_utf8_start
: Bool do
63 if self & 0x80u
8 == 0u8
then return true
64 if self & 0b1110_0000u
8 == 0b1100_0000u
8 then return true
65 if self & 0b1111_0000u
8 == 0b1110_0000u
8 then return true
66 if self & 0b1111_1000u
8 == 0b1111_0000u
8 then return true
72 # Returns the code_point from a utf16 surrogate pair
74 # assert 0xD83DDE02.from_utf16_surr == 0x1F602
75 fun from_utf16_surr
: Int do
76 var hi
= (self & 0xFFFF0000) >> 16
77 var lo
= self & 0xFFFF
79 cp
+= (hi
- 0xD800) << 10
86 # Native strings are simple C char *
87 extern class NativeString `{ char* `}
88 # Creates a new NativeString with a capacity of `length
`
89 new(length: Int) is intern
91 # Returns a char* starting at `index
`.
93 # WARNING: Unsafe for extern code, use only for temporary
94 # pointer manipulation purposes (e.g. write to file or such)
95 fun fast_cstring(index: Int): NativeString is intern
97 # Get char at `index
`.
98 fun [](index: Int): Byte is intern
100 # Set char `item
` at index.
101 fun []=(index: Int, item: Byte) is intern
103 # Copy `self` to `dest
`.
104 fun copy_to(dest: NativeString, length: Int, from: Int, to: Int) is intern
106 redef fun ==(o) is intern do return is_same_instance(o)
108 redef fun !=(o) is intern do return not is_same_instance(o)
110 # Position of the first nul character.
111 fun cstring_length: Int
114 while self[l] != 0u8 do l += 1
118 # Parse `self` as an Int.
119 fun atoi: Int is intern
121 # Parse `self` as a Float.
122 fun atof: Float `{ return atof(self); `}
124 # Gets the UTF-8 char at index `pos`
126 # Index is expressed in Unicode chars
129 # assert "かきく".as(FlatString).items.char_at(0) == 'か'
132 # If the char at position pos is an invalid Unicode char,
133 # the Unicode replacement character � (0xFFFD) will be used.
136 # assert "かきく".as(FlatString).items.char_at(1) == '�'
138 fun char_at
(pos
: Int): Char do
140 if c
& 0x80u
8 == 0u8
then return c
.ascii
141 var b
= fetch_4_hchars
(pos
)
143 if b
& 0xC00000 != 0x800000 then return 0xFFFD.code_point
144 if b
& 0xE0000000 == 0xC0000000 then
145 ret
|= (b
& 0x1F000000) >> 18
146 ret
|= (b
& 0x3F0000) >> 16
147 return ret
.code_point
149 if not b
& 0xC000 == 0x8000 then return 0xFFFD.code_point
150 if b
& 0xF0000000 == 0xE0000000 then
151 ret
|= (b
& 0xF000000) >> 12
152 ret
|= (b
& 0x3F0000) >> 10
153 ret
|= (b
& 0x3F00) >> 8
154 return ret
.code_point
156 if not b
& 0xC0 == 0x80 then return 0xFFFD.code_point
157 if b
& 0xF8000000 == 0xF0000000 then
158 ret
|= (b
.to_i
& 0x7000000) >> 6
159 ret
|= (b
.to_i
& 0x3F0000) >> 4
160 ret
|= (b
.to_i
& 0x3F00) >> 2
162 return ret
.code_point
164 return 0xFFFD.code_point
167 # Gets the byte index of char at position `n` in UTF-8 String
168 fun char_to_byte_index
(n
: Int): Int do return char_to_byte_index_cached
(n
, 0, 0)
170 # Gets the length of the character at position `pos` (1 if invalid sequence)
171 fun length_of_char_at
(pos
: Int): Int do
173 if c
& 0x80u
8 == 0x00u
8 then
175 else if c
& 0xE0u
8 == 0xC0u
8 and self[pos
+ 1] & 0xC0u
8 == 0x80u
8 then
177 else if c
& 0xF0u
8 == 0xE0u
8 and self[pos
+ 1] & 0xC0u
8 == 0x80u
8 and self[pos
+ 2] & 0xC0u
8 == 0x80u
8 then
179 else if c
& 0xF8u
8 == 0xF0u
8 and self[pos
+ 1] & 0xC0u
8 == 0x80u
8 and self[pos
+ 2] & 0xC0u
8 == 0x80u
8 and self[pos
+ 3] & 0xC0u
8 == 0x80u
8 then
186 # Gets the byte index of char at position `n` in UTF-8 String
188 # `char_from` and `byte_from` are cached values to seek from.
190 # NOTE: char_from and byte_from are not guaranteed to be valid cache values
191 # It it up to the client to ensure the validity of the information
192 fun char_to_byte_index_cached
(n
, char_from
, byte_from
: Int): Int do
200 var i
= fetch_4_chars
(ns_i
)
201 if i
& 0x80808080 != 0 then break
206 if dist
== 0 then break
207 ns_i
+= length_of_char_at
(ns_i
)
214 var i
= fetch_4_chars
(ns_i
- 4)
215 if i
& 0x80808080 != 0 then break
220 if dist
== 0 then break
221 ns_i
= find_beginning_of_char_at
(ns_i
- 1)
229 # Gets the char index of byte at position `n` in a UTF-8 String
231 # `char_from` and `byte_from` are cached values to seek from.
233 # NOTE: char_from and byte_from are not guaranteed to be valid cache values
234 # It it up to the client to ensure the validity of the information
235 fun byte_to_char_index_cached
(n
, char_from
, byte_from
: Int): Int do
240 ns_i
+= length_of_char_at
(ns_i
)
245 ns_i
= find_beginning_of_char_at
(ns_i
- 1)
252 # Returns the beginning position of the char at position `pos`
254 # If the char is invalid UTF-8, `pos` is returned as-is
257 # assert "abc".items.find_beginning_of_char_at(2) == 2
258 # assert "か".items.find_beginning_of_char_at(1) == 0
259 # assert [0x41u8, 233u8].to_s.items.find_beginning_of_char_at(1) == 1
261 fun find_beginning_of_char_at
(pos
: Int): Int do
264 if c
& 0x80u
8 == 0x00u
8 then return pos
265 while c
& 0xC0u
8 == 0x80u
8 do
270 if length_of_char_at
(stpos
) >= (endpos
- stpos
+ 1) then return pos
274 # Number of UTF-8 characters in `self` starting at `from`, for a length of `byte_length`
275 fun utf8_length
(from
, byte_length
: Int): Int is intern do
278 while byte_length
> 0 do
279 while byte_length
>= 4 do
280 var i
= fetch_4_chars
(st
)
281 if i
& 0x80808080 != 0 then break
286 if byte_length
== 0 then break
287 var cln
= length_of_char_at
(st
)
295 # Fetch 4 chars in `self` at `pos`
296 fun fetch_4_chars
(pos
: Int): Int is intern `{ return (long)*((uint32_t*)(self+pos)); `}
298 # Fetch 4 chars in `self` at `pos
`
299 fun fetch_4_hchars(pos: Int): Int is intern `{ return (long)be32toh(*((uint32_t*)(self+pos))); `}
302 # Right shifts `len` bytes of `self` from `sh` bytes starting at position `pos`
303 fun rshift
(sh
, len
, pos
: Int) do
304 copy_to
(self, len
, pos
, pos
+ sh
)
307 # Left shifts `len` bytes of `self` from `sh` bytes starting at position `pos`
308 fun lshift
(sh
, len
, pos
: Int) do
309 copy_to
(self, len
, pos
, pos
- sh
)