1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # This file is free software, which comes along with NIT. This software is
4 # distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
5 # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
6 # PARTICULAR PURPOSE. You can modify it is you want, provided this header
7 # is kept unaltered, and a notification of the changes is added.
8 # You are allowed to redistribute it and sell it, alone or is a part of
11 # Native structures for text and bytes
23 #include <libkern/OSByteOrder.h>
24 #define be32toh(x) OSSwapBigToHostInt32(x)
27 #define be32toh(val) _byteswap_ulong(val)
31 #define be16toh(val) (((val) >> 8) | ((val) << 8))
32 #define be32toh(val) ((be16toh((val) << 16) | (be16toh((val) >> 16))))
35 #define be32toh(val) betoh32(val)
40 # Gives the length of the UTF-8 char starting with `self`
42 if self & 0b1000_0000u
8 == 0u8
then
44 else if self & 0b1110_0000u
8 == 0b1100_0000u
8 then
46 else if self & 0b1111_0000u
8 == 0b1110_0000u
8 then
48 else if self & 0b1111_1000u
8 == 0b1111_0000u
8 then
55 # Is `self` a valid UTF-8 sequence start ?
58 # assert 0u8.is_valid_utf8_start
59 # assert 0xC0u8.is_valid_utf8_start
60 # assert 0xE0u8.is_valid_utf8_start
61 # assert 0xF0u8.is_valid_utf8_start
63 fun is_valid_utf8_start
: Bool do
64 if self & 0x80u
8 == 0u8
then return true
65 if self & 0b1110_0000u
8 == 0b1100_0000u
8 then return true
66 if self & 0b1111_0000u
8 == 0b1110_0000u
8 then return true
67 if self & 0b1111_1000u
8 == 0b1111_0000u
8 then return true
73 # Returns the code_point from a utf16 surrogate pair
75 # assert 0xD83DDE02u32.from_utf16_surr == 0x1F602u32
76 fun from_utf16_surr
: UInt32 do
77 var hi
= (self & 0xFFFF0000u
32) >> 16
78 var lo
= self & 0xFFFFu
32
80 cp
+= (hi
- 0xD800u
32) << 10
86 # The character which code point (unicode-wise) is `self`
88 # assert 65u32.code_point == 'A'
89 # assert 10u32.code_point == '\n'
90 # assert 0x220Bu32.code_point == '∋'
91 fun code_point
: Char `{ return self; `}
96 # Used as underlying implementation for `String` and some other `Text`.
97 extern class CString `{ char* `}
98 # Create a new `CString` with the capacity for `length` characters
99 new(length
: Int) is intern
101 # Get a char* starting at `index`.
103 # WARNING: Unsafe for extern code, use only for temporary
104 # pointer manipulation purposes (e.g. write to file or such)
105 fun fast_cstring
(index
: Int): CString is intern
107 # Get char at `index`.
108 fun [](index
: Int): Byte is intern
110 # Set char `item` at index.
111 fun []=(index
: Int, item
: Byte) is intern
113 # Copy `self` to `dest`.
114 fun copy_to
(dest
: CString, length
: Int, from
: Int, to
: Int) is intern
116 redef fun ==(o
) is intern do return is_same_instance
(o
)
118 redef fun !=(o
) is intern do return not is_same_instance
(o
)
120 # Position of the first nul character.
121 fun cstring_length
: Int
124 while self[l
] != 0u8
do l
+= 1
128 # Parse `self` as an Int.
129 fun atoi
: Int is intern
131 # Parse `self` as a Float.
132 fun atof
: Float `{ return atof(self); `}
134 # Gets the UTF-8 char at index `pos
`
136 # Index is expressed in Unicode chars
139 # assert "かきく".as(FlatString).items.char_at(0) == 'か'
142 # If the char at position pos is an invalid Unicode char,
143 # the Unicode replacement character � (0xFFFD) will be used.
146 # assert "かきく".as(FlatString).items.char_at(1) == '�'
148 fun char_at(pos: Int): Char do
150 if c & 0x80u8 == 0u8 then return c.ascii
151 var b = fetch_4_hchars(pos)
153 if b & 0xC00000u32 != 0x800000u32 then return 0xFFFD.code_point
154 if b & 0xE0000000u32 == 0xC0000000u32 then
155 ret |= (b & 0x1F000000u32) >> 18
156 ret |= (b & 0x3F0000u32) >> 16
157 return ret.code_point
159 if not b & 0xC000u32 == 0x8000u32 then return 0xFFFD.code_point
160 if b & 0xF0000000u32 == 0xE0000000u32 then
161 ret |= (b & 0xF000000u32) >> 12
162 ret |= (b & 0x3F0000u32) >> 10
163 ret |= (b & 0x3F00u32) >> 8
164 return ret.code_point
166 if not b & 0xC0u32 == 0x80u32 then return 0xFFFD.code_point
167 if b & 0xF8000000u32 == 0xF0000000u32 then
168 ret |= (b & 0x7000000u32) >> 6
169 ret |= (b & 0x3F0000u32) >> 4
170 ret |= (b & 0x3F00u32) >> 2
172 return ret.code_point
174 return 0xFFFD.code_point
177 # Gets the byte index of char at position `n
` in UTF-8 String
178 fun char_to_byte_index(n: Int): Int do return char_to_byte_index_cached(n, 0, 0)
180 # Gets the length of the character at position `pos
` (1 if invalid sequence)
181 fun length_of_char_at(pos: Int): Int do
183 if c & 0x80u8 == 0x00u8 then
185 else if c & 0xE0u8 == 0xC0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 then
187 else if c & 0xF0u8 == 0xE0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 and self[pos + 2] & 0xC0u8 == 0x80u8 then
189 else if c & 0xF8u8 == 0xF0u8 and self[pos + 1] & 0xC0u8 == 0x80u8 and self[pos + 2] & 0xC0u8 == 0x80u8 and self[pos + 3] & 0xC0u8 == 0x80u8 then
196 # Gets the byte index of char at position `n
` in UTF-8 String
198 # `char_from
` and `byte_from
` are cached values to seek from.
200 # NOTE: char_from and byte_from are not guaranteed to be valid cache values
201 # It it up to the client to ensure the validity of the information
202 fun char_to_byte_index_cached(n, char_from, byte_from: Int): Int do
210 var i = fetch_4_chars(ns_i)
211 if i & 0x80808080u32 != 0u32 then break
216 if dist == 0 then break
217 ns_i += length_of_char_at(ns_i)
224 var i = fetch_4_chars(ns_i - 4)
225 if i & 0x80808080u32 != 0u32 then break
230 if dist == 0 then break
231 ns_i = find_beginning_of_char_at(ns_i - 1)
239 # Gets the char index of byte at position `n
` in a UTF-8 String
241 # `char_from
` and `byte_from
` are cached values to seek from.
243 # NOTE: char_from and byte_from are not guaranteed to be valid cache values
244 # It it up to the client to ensure the validity of the information
245 fun byte_to_char_index_cached(n, char_from, byte_from: Int): Int do
250 ns_i += length_of_char_at(ns_i)
255 ns_i = find_beginning_of_char_at(ns_i - 1)
262 # Returns the beginning position of the char at position `pos
`
264 # If the char is invalid UTF-8, `pos
` is returned as-is
267 # assert "abc".items.find_beginning_of_char_at(2) == 2
268 # assert "か".items.find_beginning_of_char_at(1) == 0
269 # assert [0x41u8, 233u8].to_s.items.find_beginning_of_char_at(1) == 1
271 fun find_beginning_of_char_at(pos: Int): Int do
274 if c & 0x80u8 == 0x00u8 then return pos
275 while c & 0xC0u8 == 0x80u8 do
280 if length_of_char_at(stpos) >= (endpos - stpos + 1) then return pos
284 # Number of UTF-8 characters in `self` starting at `from
`, for a length of `byte_length
`
285 fun utf8_length(from, byte_length: Int): Int is intern do
288 while byte_length > 0 do
289 while byte_length >= 4 do
290 var i = fetch_4_chars(st)
291 if i & 0x80808080u32 != 0u32 then break
296 if byte_length == 0 then break
297 var cln = length_of_char_at(st)
305 # Fetch 4 chars in `self` at `pos
`
306 fun fetch_4_chars(pos: Int): UInt32 is intern `{ return *((uint32_t*)(self+pos)); `}
308 # Fetch 4 chars in `self` at `pos`
309 fun fetch_4_hchars
(pos
: Int): UInt32 is intern `{ return (uint32_t)be32toh(*((uint32_t*)(self+pos))); `}
311 # Right shifts `len
` bytes of `self` from `sh
` bytes starting at position `pos
`
312 fun rshift(sh, len, pos: Int) do
313 copy_to(self, len, pos, pos + sh)
316 # Left shifts `len
` bytes of `self` from `sh
` bytes starting at position `pos
`
317 fun lshift(sh, len, pos: Int) do
318 copy_to(self, len, pos, pos - sh)