1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Introduces UTF-8 as internal encoding for Strings in Nit.
18 intrude import standard
::string
19 intrude import standard
::file
27 #define IS_BIG_ENDIAN (*(uint16_t *)"\0\xff" < 0x100)
31 # UTF-8 char as defined in RFC-3629, e.g. 1-4 Bytes
32 extern class UnicodeChar `{ uint32_t* `}
35 redef type OTHER: UnicodeChar
37 # Transforms a byte-variable char* character to its uint32_t equivalent
38 new from_ns(ns: NativeString, index: Int) `{
39 unsigned char
* ret
= calloc
(1,4);
40 if((ns
[index
] & 0x80) == 0){ memcpy(ret + 3, ns + index, 1); }
41 else if((ns
[index
] & 0xE0) == 0xC0) { memcpy(ret + 2, ns + index, 2); }
42 else if((ns
[index
] & 0xF0) == 0xE0) { memcpy(ret + 1, ns + index, 3); }
43 else if((ns
[index
] & 0xF7) == 0xF0) { memcpy(ret, ns + index, 4); }
44 else{ memcpy(ret + 3, ns + index, 1);}
46 uint32_t tmp
= ntohl
(*((uint32_t
*)ret
));
49 return (uint32_t
*)ret
;
52 # Real length of the char in UTF8
54 # As per the specification :
57 # Length | UTF-8 octet sequence
59 # ---------+-------------------------------------------------
61 # 2 | 110xxxxx 10xxxxxx
62 # 3 | 1110xxxx 10xxxxxx 10xxxxxx
63 # 4 | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
67 if(s
<= 127) {return 1;}
68 if(s
>= 49280 && s
<= 57279) {return 2;}
69 if(s
>= 14712960 && s
<= 15712191) {return 3;}
70 if(s
>= 4034953344 && s
<= 4156538815) { return 4; }
75 # Returns the Unicode code point representing the character
77 # Note : A unicode character might not be a visible glyph, but it will be used to determine canonical equivalence
78 fun code_point: Int import UnicodeChar.len `{
81 switch
(UnicodeChar_len(self)){
86 ret
= 0 | ((val
& 0x00001F00) >> 2) | (val
& 0x0000003F);
89 ret
= 0 | ((val
& 0x000F0000) >> 4) | ((val
& 0x00003F00) >> 2) | (val
& 0x0000003F);
92 ret
= 0 | ((val
& 0x07000000) >> 6) | ((val
& 0x003F0000) >> 4) | ((val
& 0x00003F00) >> 2) | (val
& 0x0000003F);
95 unsigned char
* rt
= (unsigned char
*) &ret
;
99 # Warning : This does not follow the Unicode specification for now
101 # TODO: Support Unicode-compliant comparison
102 redef fun <(o) do return self.code_point < o.code_point
104 # Returns an upper-case version of self
106 # NOTE : Works only on ASCII chars
107 # TODO : Support unicode for to_upper
108 fun to_upper: UnicodeChar import UnicodeChar.code_point `{
109 if(*self < 97 || *self > 122){ return self; }
110 uint32_t
* ret
= calloc
(1,4);
115 # Returns an lower-case version of self
117 # NOTE : Works only on ASCII chars
118 # TODO : Support unicode for to_upper
119 fun to_lower: UnicodeChar import UnicodeChar.code_point `{
120 if(*self < 65 || *self > 90){ return self; }
121 uint32_t
* ret
= calloc
(1,4);
128 if not o isa UnicodeChar then return false
129 if o.code_point == self.code_point then return true
133 redef fun output import UnicodeChar.len `{
134 uint32_t self0
= *self;
136 uint32_t tmp
= ntohl
(self0
);
137 memcpy
(&self0
, &tmp
, 4);
139 unsigned char
* s
= (unsigned char
*) &self0
;
140 switch
(UnicodeChar_len(self0
)){
145 printf
("%c%c", s
[2], s
[3]);
148 printf
("%c%c%c", s
[1], s
[2], s
[3]);
151 printf
("%c%c%c%c", s
[0], s
[1], s
[2], s
[3]);
156 redef fun to_s: FlatString import FlatString.full, UnicodeChar.len `{
157 int len
= UnicodeChar_len(self);
158 char
* r
= malloc
(len
+ 1);
160 uint32_t src
= *self;
162 uint32_t tmp
= htonl
(src
);
163 memcpy
(&src
, &tmp
, 4);
165 unsigned char
* s
= (unsigned char
*) &src
;
167 case
1: memcpy
(r
, s
+3, 1); break;
168 case
2: memcpy
(r
, s
+2, 2); break;
169 case
3: memcpy
(r
, s
+1, 3); break;
170 case
4: memcpy
(r
, s
, 4); break;
172 return new_FlatString_full
(r
, 0, len
- 1, len
, 1);
176 # Used to keep track of the last accessed char in a String
178 # The position (as in char) of a String
180 # The position in the NativeString underlying the String
184 class FlatStringReviter
185 super IndexedIterator[UnicodeChar]
187 # The NativeString to iterate upon
188 private var ns: NativeString
190 # The position in the string
193 # The position in the native string
194 private var bytepos: Int
196 init(s: FlatString) do from(s, s.length - 1)
198 init from(s: FlatString, position: Int)
202 bytepos = s.byte_index(position)
208 while ns[bytepos].ascii.bin_and(0xC0) == 0x80 do
214 redef fun index do return pos
216 redef fun item do return new UnicodeChar.from_ns(ns, bytepos)
218 redef fun is_ok do return pos >= 0
222 super IndexedIterator[UnicodeChar]
224 private var ns: NativeString
228 private var bytepos: Int
230 private var slen: Int
232 private var it: UnicodeChar
234 private var is_created = false
236 init(s: FlatString) do from(s, 0)
238 init from(s: FlatString, position: Int) do
241 bytepos = s.byte_index(position)
245 redef fun index do return pos
247 redef fun is_ok do return pos < slen
250 if not is_created then
251 it = new UnicodeChar.from_ns(ns, bytepos)
259 if not is_created then
260 it = new UnicodeChar.from_ns(ns, bytepos)
269 redef class FlatString
271 redef type OTHER: FlatString
273 # Length in bytes of the string (e.g. the length of the C string)
274 redef var bytelen: Int
276 # Cache for the last accessed character in the char
277 var cache = new CharCache(-1,-1)
279 redef var length = length_l is lazy
281 private init full(items: NativeString, from, to, bytelen, len: Int)
286 self.bytelen = bytelen
290 # Length implementation
291 private fun length_l: Int import FlatString.items, FlatString.index_to, FlatString.index_from `{
292 char
* ns
= FlatString_items(self);
293 int i
= FlatString_index_from(self);
294 int max
= FlatString_index_to(self);
298 if((c
& 0x80) == 0) { i+= 1; }
299 else if((c
& 0xE0) == 0xC0) { i += 2; }
300 else if((c
& 0xF0) == 0xE0) { i += 3; }
301 else if((c
& 0xF7) == 0xF0) { i += 4; }
312 for i in [0 .. length[ do
313 if o_pos >= olen then return false
314 if char_at(i) > o.char_at(i) then return false
315 if char_at(i) < o.char_at(i) then return true
321 if o == null then return false
322 if not o isa FlatString then return super
324 var itslen = o.length
325 if mylen != itslen then return false
329 while mypos < mylen do
330 if char_at(mypos) != o.char_at(itspos) then return false
337 private fun byte_index(index: Int): Int do
339 assert index < length
341 # Find best insertion point
342 var delta_begin = index
343 var delta_end = (length - 1) - index
344 var delta_cache = (cache.position - index).abs
345 var min = delta_begin
347 if delta_cache < min then min = delta_cache
348 if delta_end < min then min = delta_end
354 if min == delta_begin then
357 else if min == delta_cache then
359 my_i = cache.position
365 while my_i < index do
366 if myits[ns_i].ascii.bin_and(0x80) == 0 then
368 else if myits[ns_i].ascii.bin_and(0xE0) == 0xC0 then
370 else if myits[ns_i].ascii.bin_and(0xF0) == 0xE0 then
372 else if myits[ns_i].ascii.bin_and(0xF7) == 0xF0 then
380 while my_i > index do
381 if myits[ns_i].ascii.bin_and(0xC0) != 0x80 then
383 if my_i == index then break
388 cache.position = index
394 fun char_at(pos: Int): UnicodeChar do
395 return new UnicodeChar.from_ns(items, byte_index(pos))
398 private init with_bytelen(items: NativeString, index_from: Int, index_to: Int, bytelen: Int) do
400 self.index_from = index_from
401 self.index_to = index_to
402 self.bytelen = bytelen
405 redef fun reversed do
406 var new_str = new NativeString(bytelen)
408 var my_pos = index_from
410 for i in [0..length[ do
411 var c = char_at(i).len
413 its.copy_to(new_str, c, my_pos, s_pos)
416 return new FlatString.full(new_str, 0, bytelen - 1, bytelen, length)
419 redef fun to_upper do
420 var ns = new NativeString(bytelen)
422 for i in [0 .. length[
425 c.to_upper.to_s.items.copy_to(ns, c.len, 0, offset)
428 return new FlatString.full(ns, 0, bytelen - 1, bytelen, length)
431 redef fun to_lower do
432 var ns = new NativeString(bytelen)
434 for i in [0 .. length[
437 c.to_lower.to_s.items.copy_to(ns, c.len, 0, offset)
440 return new FlatString.full(ns, 0, bytelen - 1, bytelen, length)
444 if o isa Buffer then o = o.to_s
445 if o isa FlatString then
446 var new_str = new NativeString(bytelen + o.bytelen + 1)
447 var new_bytelen = bytelen + o.bytelen
448 new_str[new_bytelen] = '\0'
449 var newlen = length + o.length
450 items.copy_to(new_str, bytelen, index_from, 0)
451 o.items.copy_to(new_str, o.bytelen, o.index_from, bytelen)
452 return new FlatString.full(new_str, 0, new_bytelen - 1, new_bytelen, newlen)
453 else if o isa Concat then
454 return new Concat(self, o)
456 # If it goes to this point, that means another String implementation was concerned, therefore you need to support the + operation for this variant
462 var mybtlen = bytelen
463 var new_bytelen = mybtlen * i
465 var newlen = mylen * i
466 var ns = new NativeString(new_bytelen + 1)
467 ns[new_bytelen] = '\0'
470 items.copy_to(ns, bytelen, index_from, offset)
474 return new FlatString.full(ns, 0, new_bytelen - 1, new_bytelen, newlen)
478 redef fun substring(from: Int, count: Int) do
483 if count < 0 then count = 0
487 if count == 0 then return empty
489 var real_from = byte_index(from)
491 var lst = from + count - 1
493 if lst > length - from then
494 return new FlatString.with_bytelen(items, real_from, index_to, index_to - real_from)
497 var real_to = byte_index(lst)
499 return new FlatString.full(items, real_from, real_to, (real_to + char_at(lst).len) - real_from, count)
502 redef fun to_cstring do
503 if real_items != null then return real_items.as(not null)
504 var new_items = new NativeString(bytelen + 1)
505 self.items.copy_to(new_items, bytelen, index_from, 0)
506 new_items[bytelen] = '\0'
507 self.real_items = new_items
514 # Length of the string, in bytes
515 fun bytelen: Int is abstract
519 redef class FlatBuffer
521 redef var bytelen: Int
523 redef init from(s) do
526 for i in s.substrings do self.append(i)
528 items = new NativeString(s.bytelen)
529 if s isa FlatString then
530 s.items.copy_to(items, s.bytelen, s.index_from, 0)
532 s.as(FlatBuffer).items.copy_to(items, s.as(FlatBuffer).bytelen, 0, 0)
539 # Replaces the char at `index
` by `item
`
540 fun char_at=(index: Int, item: UnicodeChar) do
542 if index == length then
546 assert index >= 0 and index < length
547 var ip = byte_at(index)
548 var c = char_at_byte(ip)
549 var size_diff = item.len - c.len
550 if size_diff > 0 then
551 rshift_bytes(ip + c.len, size_diff)
552 else if size_diff < 0 then
553 lshift_bytes(ip + c.len, -size_diff)
556 s.items.copy_to(items, s.bytelen, 0, ip)
559 # Shifts the content of the buffer by `len
` bytes to the right, starting at byte `from
`
560 fun rshift_bytes(from: Int, len: Int) import FlatBuffer.bytelen, FlatBuffer.bytelen=, FlatBuffer.items `{
561 long bt
= FlatBuffer_bytelen(self);
562 char
* ns
= FlatBuffer_items(self);
563 int off
= from
+ len
;
564 memmove
(ns
+ off
, ns
+ from
, bt
- from
);
565 FlatBuffer_bytelen__assign(self, bt
+ len
);
568 # Shifts the content of the buffer by `len
` bytes to the left, starting at `from
`
569 fun lshift_bytes(from: Int, len: Int) import FlatBuffer.bytelen, FlatBuffer.bytelen=, FlatBuffer.items `{
570 long bt
= FlatBuffer_bytelen(self);
571 char
* ns
= FlatBuffer_items(self);
572 int off
= from
- len
;
573 memmove
(ns
+ off
, ns
+ from
, bt
- from
);
574 FlatBuffer_bytelen__assign(self, bt
- len
);
577 # Get the Unicode char stored at `index
` in `self`
578 fun char_at(index: Int): UnicodeChar do return new UnicodeChar.from_ns(items, byte_at(index))
580 # Get the Unicode char stored at `index
` (bytewise) in `self`
581 fun char_at_byte(index: Int): UnicodeChar do return new UnicodeChar.from_ns(items, index)
583 # Add equivalent that supports Unicode
584 fun add_unicode(c: UnicodeChar) do
586 if s.bytelen + bytelen > capacity then enlarge(s.bytelen)
587 s.items.copy_to(items, s.bytelen, 0, bytelen)
590 # Gets the byte index (in NativeString) of the char stored at `i
`
591 fun byte_at(i: Int): Int do
592 assert i < length and i >= 0
596 if items[ns_i].ascii.bin_and(0x80) == 0 then
598 else if items[ns_i].ascii.bin_and(0xE0) == 0xC0 then
600 else if items[ns_i].ascii.bin_and(0xF0) == 0xE0 then
602 else if items[ns_i].ascii.bin_and(0xF7) == 0xF0 then
612 redef fun enlarge(cap) do
614 if cap <= c then return
615 while c <= cap do c = c * 2 + 2
616 var a = new NativeString(c+1)
617 if bytelen > 0 then items.copy_to(a, bytelen, 0, 0)
622 redef fun append(s) do
624 for i in s.substrings do append i
626 var i = s.as(FlatString)
628 var iblen = i.bytelen
629 var newlen = blen + iblen
630 if newlen > capacity then
633 i.items.copy_to(items, iblen, i.index_from, blen)
640 var nns = new NativeString(bytelen)
646 var c = char_at_byte(myp).len
648 ns.copy_to(nns, c, myp, itsp)
659 redef fun copy(s, l, d, ns) do
660 if not d isa FlatBuffer then
661 # This implementation here is only concerned by the FlatBuffer
662 # If you implement a new Buffer subclass, make sure to support this operation via refinement.
666 var re = byte_at(s + l - 1)
668 var rns = d.byte_at(ns)
669 items.copy_to(d.items, rl, rns, rs)
672 redef fun times(i) do
676 if newlen > capacity then enlarge(newlen)
678 items.copy_to(items, len, 0, off)
686 for i in [0 .. length[ do
688 var c = char_at_byte(pos)
690 if c == d then continue
691 d.to_s.items.copy_to(items, 1, 0, pos)
696 for i in [0 .. length[ do
698 var c = char_at_byte(pos)
700 if c == d then continue
701 d.to_s.items.copy_to(items, 1, 0, pos)
705 redef fun to_cstring do
706 var ns = new NativeString(bytelen)
707 items.copy_to(ns, bytelen, 0, 0)
712 redef class NativeString
714 redef fun to_s: FlatString
716 var len = cstring_length
717 return to_s_with_length(len)
720 redef fun to_s_with_length(len: Int): FlatString
722 return new FlatString.with_bytelen(self, 0, len - 1, len)
725 redef fun to_s_with_copy
727 var length = cstring_length
728 var new_self = new NativeString(length + 1)
729 copy_to(new_self, length, 0, 0)
730 return new FlatString.with_bytelen(new_self, 0, length - 1, length)
734 redef class FileWriter
738 if s isa FlatText then
739 write_native(s.to_cstring, s.bytelen)
740 else for i in s.substrings do write_native(i.to_cstring, i.length)