fun substring(from: Int, count: Int): SELFTYPE is abstract
# Iterates on the substrings of self if any
- fun substrings: Iterator[FlatText] is abstract
+ private fun substrings: Iterator[FlatText] is abstract
# Is the current Text empty (== "")
#
# Returns -1 if not found
#
# DEPRECATED : Use self.chars.last_index_of_from instead
- fun last_index_of_from(item: Char, pos: Int): Int
- do
- var iter = self.chars.reverse_iterator_from(pos)
- while iter.is_ok do
- if iter.item == item then return iter.index
- iter.next
- end
- return -1
- end
+ fun last_index_of_from(item: Char, pos: Int): Int do return chars.last_index_of_from(item, pos)
# Gets an iterator on the chars of self
#
# If `self` contains only digits and alpha <= 'f', return the corresponding integer.
#
# assert "ff".to_hex == 255
- fun to_hex: Int do return a_to(16)
+ fun to_hex(pos, ln: nullable Int): Int do
+ var res = 0
+ if pos == null then pos = 0
+ if ln == null then ln = length - pos
+ var max = pos + ln
+ for i in [pos .. max[ do
+ res <<= 4
+ res += self[i].from_hex
+ end
+ return res
+ end
# If `self` contains only digits <= '7', return the corresponding integer.
#
end
end
- # Returns `true` if the string contains only Numeric values (and one "," or one "." character)
+ # Is this string in a valid numeric format compatible with `to_f`?
#
# assert "123".is_numeric == true
# assert "1.2".is_numeric == true
- # assert "1,2".is_numeric == true
+ # assert "-1.2".is_numeric == true
+ # assert "-1.23e-2".is_numeric == true
# assert "1..2".is_numeric == false
+ # assert "".is_numeric == false
fun is_numeric: Bool
do
- var has_point_or_comma = false
+ var has_point = false
+ var e_index = -1
for i in [0..length[ do
var c = chars[i]
if not c.is_numeric then
- if (c == '.' or c == ',') and not has_point_or_comma then
- has_point_or_comma = true
+ if c == '.' and not has_point then
+ has_point = true
+ else if c == 'e' and e_index == -1 and i > 0 and i < length - 1 and chars[i-1] != '-' then
+ e_index = i
+ else if c == '-' and i == e_index + 1 and i < length - 1 then
else
return false
end
end
end
- return true
+ return not is_empty
end
# Returns `true` if the string contains only Hex chars
if c >= '0' and c <= '9' then
res.add('_')
- res.append(c.ascii.to_s)
+ res.append(c.code_point.to_s)
res.add('d')
start = 1
end
continue
end
if underscore then
- res.append('_'.ascii.to_s)
+ res.append('_'.code_point.to_s)
res.add('d')
end
if c >= '0' and c <= '9' then
underscore = true
else
res.add('_')
- res.append(c.ascii.to_s)
+ res.append(c.code_point.to_s)
res.add('d')
underscore = false
end
end
if underscore then
- res.append('_'.ascii.to_s)
+ res.append('_'.code_point.to_s)
res.add('d')
end
return res.to_s
# Three digits are always used to avoid following digits to be interpreted as an element
# of the octal sequence.
#
- # assert "{0.ascii}{1.ascii}{8.ascii}{31.ascii}{32.ascii}".escape_to_c == "\\000\\001\\010\\037 "
+ # assert "{0.code_point}{1.code_point}{8.code_point}{31.code_point}{32.code_point}".escape_to_c == "\\000\\001\\010\\037 "
#
# The exceptions are the common `\t` and `\n`.
fun escape_to_c: String
b.append("\\\'")
else if c == '\\' then
b.append("\\\\")
- else if c.ascii < 32 then
+ else if c.code_point < 32 then
b.add('\\')
- var oct = c.ascii.to_base(8, false)
+ var oct = c.code_point.to_base(8, false)
# Force 3 octal digits since it is the
# maximum allowed in the C specification
if oct.length == 1 then
else if c == ':' or c == ' ' or c == '#' then
b.add('\\')
b.add(c)
- else if c.ascii < 32 or c == ';' or c == '|' or c == '\\' or c == '=' then
- b.append("?{c.ascii.to_base(16, false)}")
+ else if c.code_point < 32 or c == ';' or c == '|' or c == '\\' or c == '=' then
+ b.append("?{c.code_point.to_base(16, false)}")
else
b.add(c)
end
# assert s.length == 2
# var u = s.unescape_nit
# assert u.length == 1
- # assert u.chars[0].ascii == 10 # (the ASCII value of the "new line" character)
+ # assert u.chars[0].code_point == 10 # (the ASCII value of the "new line" character)
fun unescape_nit: String
do
var res = new Buffer.with_cap(self.length)
return res.to_s
end
+ # Returns `self` with all characters escaped with their UTF-16 representation
+ #
+ # assert "Aรจใ๐".escape_to_utf16 == "\\u0041\\u00e8\\u3042\\ud800\\udfd3"
+ fun escape_to_utf16: String do
+ var buf = new Buffer
+ for i in chars do buf.append i.escape_to_utf16
+ return buf.to_s
+ end
+
+ # Returns the Unicode char escaped by `self`
+ #
+ # assert "\\u0041".from_utf16_escape == 'A'
+ # assert "\\ud800\\udfd3".from_utf16_escape == '๐'
+ # assert "\\u00e8".from_utf16_escape == 'รจ'
+ # assert "\\u3042".from_utf16_escape == 'ใ'
+ fun from_utf16_escape(pos, ln: nullable Int): Char do
+ if pos == null then pos = 0
+ if ln == null then ln = length - pos
+ if ln < 6 then return 0xFFFD.code_point
+ var cp = from_utf16_digit(pos + 2)
+ if cp < 0xD800 then return cp.code_point
+ if cp > 0xDFFF then return cp.code_point
+ if cp > 0xDBFF then return 0xFFFD.code_point
+ if ln == 6 then return 0xFFFD.code_point
+ if ln < 12 then return 0xFFFD.code_point
+ cp <<= 16
+ cp += from_utf16_digit(pos + 8)
+ var cplo = cp & 0xFFFF
+ if cplo < 0xDC00 then return 0xFFFD.code_point
+ if cplo > 0xDFFF then return 0xFFFD.code_point
+ return cp.from_utf16_surr.code_point
+ end
+
+ # Returns a UTF-16 escape value
+ #
+ # var s = "\\ud800\\udfd3"
+ # assert s.from_utf16_digit(2) == 0xD800
+ # assert s.from_utf16_digit(8) == 0xDFD3
+ fun from_utf16_digit(pos: nullable Int): Int do
+ if pos == null then pos = 0
+ return to_hex(pos, 4)
+ end
+
# Encode `self` to percent (or URL) encoding
#
# assert "aBc09-._~".to_percent_encoding == "aBc09-._~"
# assert "%()< >".to_percent_encoding == "%25%28%29%3c%20%3e"
# assert ".com/post?e=asdf&f=123".to_percent_encoding == ".com%2fpost%3fe%3dasdf%26f%3d123"
+ # assert "รฉใใใ".to_percent_encoding == "%c3%a9%e3%81%82%e3%81%84%e3%81%86"
fun to_percent_encoding: String
do
var buf = new Buffer
c == '_' or c == '~'
then
buf.add c
- else buf.append "%{c.ascii.to_hex}"
+ else
+ var bytes = c.to_s.bytes
+ for b in bytes do buf.append "%{b.to_i.to_hex}"
+ end
end
return buf.to_s
# assert "%25%28%29%3C%20%3E".from_percent_encoding == "%()< >"
# assert "incomplete %".from_percent_encoding == "incomplete ?"
# assert "invalid % usage".from_percent_encoding == "invalid ? usage"
+ # assert "%c3%a9%e3%81%82%e3%81%84%e3%81%86".from_percent_encoding == "รฉใใใ"
fun from_percent_encoding: String
do
- var buf = new Buffer
+ var len = bytelen
+ var has_percent = false
+ for c in chars do
+ if c == '%' then
+ len -= 2
+ has_percent = true
+ end
+ end
+ # If no transformation is needed, return self as a string
+ if not has_percent then return to_s
+
+ var buf = new NativeString(len)
var i = 0
+ var l = 0
while i < length do
var c = chars[i]
if c == '%' then
if i + 2 >= length then
# What follows % has been cut off
- buf.add '?'
+ buf[l] = '?'.ascii
else
i += 1
var hex_s = substring(i, 2)
if hex_s.is_hex then
var hex_i = hex_s.to_hex
- buf.add hex_i.ascii
+ buf[l] = hex_i.to_b
i += 1
else
# What follows a % is not Hex
- buf.add '?'
+ buf[l] = '?'.ascii
i -= 1
end
end
- else buf.add c
+ else buf[l] = c.ascii
i += 1
+ l += 1
end
- return buf.to_s
+ return buf.to_s_unsafe(l)
end
# Escape the characters `<`, `>`, `&`, `"`, `'` and `/` as HTML/XML entity references.
for i in [0..length[ do
var char = chars[i]
- h = (h << 5) + h + char.ascii
+ h = (h << 5) + h + char.code_point
end
hash_cache = h
return hash_cache.as(not null)
end
- # Gives the formatted string back as a Nit string with `args` in place
+ # Format `self` by replacing each `%n` with the `n`th item of `args`
+ #
+ # The character `%` followed by something other than a number are left as is.
+ # To represent a `%` followed by a number, double the `%`, as in `%%7`.
#
- # assert "This %1 is a %2.".format("String", "formatted String") == "This String is a formatted String."
- # assert "\\%1 This string".format("String") == "\\%1 This string"
+ # assert "This %0 is a %1.".format("String", "formatted String") == "This String is a formatted String."
+ # assert "Do not escape % nor %%1".format("unused") == "Do not escape % nor %1"
fun format(args: Object...): String do
var s = new Array[Text]
var curr_st = 0
var i = 0
while i < length do
- # Skip escaped characters
- if self[i] == '\\' then
- i += 1
- # In case of format
- else if self[i] == '%' then
+ if self[i] == '%' then
var fmt_st = i
i += 1
var ciph_st = i
while i < length and self[i].is_numeric do
i += 1
end
- i -= 1
- var fmt_end = i
- var ciph_len = fmt_end - ciph_st + 1
- var arg_index = substring(ciph_st, ciph_len).to_i - 1
+ var ciph_len = i - ciph_st
+ if ciph_len == 0 then
+ # What follows '%' is not a number.
+ s.push substring(curr_st, i - curr_st)
+ if i < length and self[i] == '%' then
+ # Skip the next `%`
+ i += 1
+ end
+ curr_st = i
+ continue
+ end
+
+ var arg_index = substring(ciph_st, ciph_len).to_i
if arg_index >= args.length then continue
s.push substring(curr_st, fmt_st - curr_st)
s.push args[arg_index].to_s
- curr_st = i + 1
+
+ curr_st = i
+ i -= 1
end
i += 1
end
return s.plain_to_s
end
+ # Return the Levenshtein distance between two strings
+ #
+ # ~~~
+ # assert "abcd".levenshtein_distance("abcd") == 0
+ # assert "".levenshtein_distance("abcd") == 4
+ # assert "abcd".levenshtein_distance("") == 4
+ # assert "abcd".levenshtein_distance("xyz") == 4
+ # assert "abcd".levenshtein_distance("xbdy") == 3
+ # ~~~
+ fun levenshtein_distance(other: String): Int
+ do
+ var slen = self.length
+ var olen = other.length
+
+ # fast cases
+ if slen == 0 then return olen
+ if olen == 0 then return slen
+ if self == other then return 0
+
+ # previous row of distances
+ var v0 = new Array[Int].with_capacity(olen+1)
+
+ # current row of distances
+ var v1 = new Array[Int].with_capacity(olen+1)
+
+ for j in [0..olen] do
+ # prefix insert cost
+ v0[j] = j
+ end
+
+ for i in [0..slen[ do
+
+ # prefix delete cost
+ v1[0] = i + 1
+
+ for j in [0..olen[ do
+ # delete cost
+ var cost1 = v1[j] + 1
+ # insert cost
+ var cost2 = v0[j + 1] + 1
+ # same char cost (+0)
+ var cost3 = v0[j]
+ # change cost
+ if self[i] != other[j] then cost3 += 1
+ # keep the min
+ v1[j+1] = cost1.min(cost2).min(cost3)
+ end
+
+ # Switch columns:
+ # * v1 become v0 in the next iteration
+ # * old v0 is reused as the new v1
+ var tmp = v1
+ v1 = v0
+ v0 = tmp
+ end
+
+ return v0[olen]
+ end
+
# Copies `n` bytes from `self` at `src_offset` into `dest` starting at `dest_offset`
#
# Basically a high-level synonym of NativeString::copy_to
#
# var ns = new NativeString(8)
# "Text is String".copy_to_native(ns, 8, 2, 0)
- # assert ns.to_s_with_length(8) == "xt is St"
+ # assert ns.to_s_unsafe(8) == "xt is St"
#
fun copy_to_native(dest: NativeString, n, src_offset, dest_offset: Int) do
var mypos = src_offset
#
# Warning : Might be void in some subclasses, be sure to check
# if set before using it.
- private var items: NativeString is noinit
-
- # Real items, used as cache for to_cstring is called
- private var real_items: nullable NativeString = null
+ var items: NativeString is noinit
# Returns a char* starting at position `first_byte`
#
#
# As always, do not modify the content of the String in C code, if this is what you want
# copy locally the char* as Nit Strings are immutable.
- private fun fast_cstring: NativeString is abstract
+ fun fast_cstring: NativeString is abstract
redef var length = 0
redef fun is_empty do return target.is_empty
- redef fun length do return target.length
+ redef fun length do return target.bytelen
redef fun iterator do return self.iterator_from(0)
var ns = new NativeString(nslen + 1)
ns[nslen] = 0u8
native_byte_to_s(ns, nslen + 1)
- return ns.to_s_with_length(nslen)
+ return ns.to_s_unsafe(nslen)
end
end
redef class Int
# Wrapper of strerror C function
- private fun strerror_ext: NativeString `{ return strerror(self); `}
+ private fun strerror_ext: NativeString `{ return strerror((int)self); `}
# Returns a string describing error number
fun strerror: String do return strerror_ext.to_s
redef class Char
+ # Returns a sequence with the UTF-8 bytes of `self`
+ #
+ # assert 'a'.bytes == [0x61u8]
+ # assert 'ใพ'.bytes == [0xE3u8, 0x81u8, 0xBEu8]
+ fun bytes: SequenceRead[Byte] do return to_s.bytes
+
+ # Is `self` an UTF-16 surrogate pair ?
+ fun is_surrogate: Bool do
+ var cp = code_point
+ return cp >= 0xD800 and cp <= 0xDFFF
+ end
+
# Length of `self` in a UTF-8 String
private fun u8char_len: Int do
- var c = self.ascii
+ var c = self.code_point
if c < 0x80 then return 1
if c <= 0x7FF then return 2
if c <= 0xFFFF then return 3
var ln = u8char_len
var ns = new NativeString(ln + 1)
u8char_tos(ns, ln)
- return ns.to_s_with_length(ln)
+ return ns.to_s_unsafe(ln)
+ end
+
+ # Returns `self` escaped to UTF-16
+ #
+ # i.e. Represents `self`.`code_point` using UTF-16 codets escaped
+ # with a `\u`
+ #
+ # assert 'A'.escape_to_utf16 == "\\u0041"
+ # assert 'รจ'.escape_to_utf16 == "\\u00e8"
+ # assert 'ใ'.escape_to_utf16 == "\\u3042"
+ # assert '๐'.escape_to_utf16 == "\\ud800\\udfd3"
+ fun escape_to_utf16: String do
+ var cp = code_point
+ var buf: Buffer
+ if cp < 0xD800 or (cp >= 0xE000 and cp <= 0xFFFF) then
+ buf = new Buffer.with_cap(6)
+ buf.append("\\u0000")
+ var hx = cp.to_hex
+ var outid = 5
+ for i in hx.chars.reverse_iterator do
+ buf[outid] = i
+ outid -= 1
+ end
+ else
+ buf = new Buffer.with_cap(12)
+ buf.append("\\u0000\\u0000")
+ var lo = (((cp - 0x10000) & 0x3FF) + 0xDC00).to_hex
+ var hi = ((((cp - 0x10000) & 0xFFC00) >> 10) + 0xD800).to_hex
+ var out = 2
+ for i in hi do
+ buf[out] = i
+ out += 1
+ end
+ out = 8
+ for i in lo do
+ buf[out] = i
+ out += 1
+ end
+ end
+ return buf.to_s
end
private fun u8char_tos(r: NativeString, len: Int) `{
return (self >= 'a' and self <= 'z') or (self >= 'A' and self <= 'Z')
end
+ # Is `self` an hexadecimal digit ?
+ #
+ # assert 'A'.is_hexdigit
+ # assert not 'G'.is_hexdigit
+ # assert 'a'.is_hexdigit
+ # assert not 'g'.is_hexdigit
+ # assert '5'.is_hexdigit
+ fun is_hexdigit: Bool do return (self >= '0' and self <= '9') or (self >= 'A' and self <= 'F') or
+ (self >= 'a' and self <= 'f')
+
# Returns true if the char is an alpha or a numeric digit
#
# assert 'a'.is_alphanumeric
do
return self.is_numeric or self.is_alpha
end
+
+ # Returns `self` to its int value
+ #
+ # REQUIRE: `is_hexdigit`
+ fun from_hex: Int do
+ if self >= '0' and self <= '9' then return code_point - 0x30
+ if self >= 'A' and self <= 'F' then return code_point - 0x37
+ if self >= 'a' and self <= 'f' then return code_point - 0x57
+ # Happens if self is not a hexdigit
+ assert self.is_hexdigit
+ # To make flow analysis happy
+ abort
+ end
end
redef class Collection[E]
# Returns `self` as a String of `length`.
fun to_s_with_length(length: Int): String is abstract
+ # Returns a new instance of `String` with self as `_items`
+ #
+ # /!\: Does not clean the items for compliance with UTF-8,
+ # Use only if you know what you are doing
+ fun to_s_unsafe(len: nullable Int): String is abstract
+
# Returns `self` as a String with `bytelen` and `length` set
#
# SEE: `abstract_text::Text` for more infos on the difference
nitlanguage / nit.git / blobdiff