Merge: Faster hex parsing

As said in #1895, we need faster parsing of UTF-16 escaping sequences, this PR is the answer.

It makes the runtime of the `large_escaped` benchmark go down from ~5s to ~3.5s, and with valgrind, from 26GIr to 20GIr

Note: based on #1886, only the 4 last commits are of interest here

Pull-Request: #1896
Reviewed-by: Jean Privat <jean@pryen.org>

diff --git a/contrib/pep8analysis/src/model/operands.nit b/contrib/pep8analysis/src/model/operands.nit
index d74d1f1..8723a4a 100644 (file)
--- a/contrib/pep8analysis/src/model/operands.nit
+++ b/contrib/pep8analysis/src/model/operands.nit
@@ -33,8 +33,7 @@ redef class AStringValue
 end
 
 redef class AHexValue
-       #TODO
-       redef fun to_i do return n_hex.text.to_hex
+       redef fun to_i do return n_hex.text.trim.to_hex(2)
 end
 
 redef class TString # TkBlock

diff --git a/lib/core/text/abstract_text.nit b/lib/core/text/abstract_text.nit
index d4873ad..fc8ec5b 100644 (file)
--- a/lib/core/text/abstract_text.nit
+++ b/lib/core/text/abstract_text.nit
@@ -248,7 +248,17 @@ abstract class Text
        # If `self` contains only digits and alpha <= 'f', return the corresponding integer.
        #
        #     assert "ff".to_hex == 255
-       fun to_hex: Int do return a_to(16)
+       fun to_hex(pos, ln: nullable Int): Int do
+               var res = 0
+               if pos == null then pos = 0
+               if ln == null then ln = length - pos
+               var max = pos + ln
+               for i in [pos .. max[ do
+                       res <<= 4
+                       res += self[i].from_hex
+               end
+               return res
+       end
 
        # If `self` contains only digits <= '7', return the corresponding integer.
        #
@@ -733,21 +743,32 @@ abstract class Text
        #     assert "\\ud800\\udfd3".from_utf16_escape == '𐏓'
        #     assert "\\u00e8".from_utf16_escape == 'è'
        #     assert "\\u3042".from_utf16_escape == 'あ'
-       fun from_utf16_escape: Char do
-               var ln = length
-               if ln != 6 and ln != 12 then return 0xFFFD.code_point
-               var cphi = substring(2, 4).to_hex
-               if cphi < 0xD800 then return cphi.code_point
-               if cphi > 0xDFFF then return cphi.code_point
-               if cphi > 0xDBFF then return 0xFFFD.code_point
-               var cp = 0
-               cp += (cphi - 0xD800) << 10
-               var cplo = substring(8, 4).to_hex
+       fun from_utf16_escape(pos, ln: nullable Int): Char do
+               if pos == null then pos = 0
+               if ln == null then ln = length - pos
+               if ln < 6 then return 0xFFFD.code_point
+               var cp = from_utf16_digit(pos + 2)
+               if cp < 0xD800 then return cp.code_point
+               if cp > 0xDFFF then return cp.code_point
+               if cp > 0xDBFF then return 0xFFFD.code_point
+               if ln == 6 then return 0xFFFD.code_point
+               if ln < 12 then return 0xFFFD.code_point
+               cp <<= 16
+               cp += from_utf16_digit(pos + 8)
+               var cplo = cp & 0xFFFF
                if cplo < 0xDC00 then return 0xFFFD.code_point
                if cplo > 0xDFFF then return 0xFFFD.code_point
-               cp += cplo - 0xDC00
-               cp += 0x10000
-               return cp.code_point
+               return cp.from_utf16_surr.code_point
+       end
+
+       # Returns a UTF-16 escape value
+       #
+       #     var s = "\\ud800\\udfd3"
+       #     assert s.from_utf16_digit(2) == 0xD800
+       #     assert s.from_utf16_digit(8) == 0xDFD3
+       fun from_utf16_digit(pos: nullable Int): Int do
+               if pos == null then pos = 0
+               return to_hex(pos, 4)
        end
 
        # Encode `self` to percent (or URL) encoding
@@ -1659,6 +1680,12 @@ redef class Char
        #     assert 'ま'.bytes == [0xE3u8, 0x81u8, 0xBEu8]
        fun bytes: SequenceRead[Byte] do return to_s.bytes
 
+       # Is `self` an UTF-16 surrogate pair ?
+       fun is_surrogate: Bool do
+               var cp = code_point
+               return cp >= 0xD800 and cp <= 0xDFFF
+       end
+
        # Length of `self` in a UTF-8 String
        private fun u8char_len: Int do
                var c = self.code_point
@@ -1791,6 +1818,19 @@ redef class Char
        do
                return self.is_numeric or self.is_alpha
        end
+
+       # Returns `self` to its int value
+       #
+       # REQUIRE: `is_hexdigit`
+       fun from_hex: Int do
+               if self >= '0' and self <= '9' then return code_point - 0x30
+               if self >= 'A' and self <= 'F' then return code_point - 0x37
+               if self >= 'a' and self <= 'f' then return code_point - 0x57
+               # Happens if self is not a hexdigit
+               assert self.is_hexdigit
+               # To make flow analysis happy
+               abort
+       end
 end
 
 redef class Collection[E]

diff --git a/lib/core/text/flat.nit b/lib/core/text/flat.nit
index c3332d0..b2336e6 100644 (file)
--- a/lib/core/text/flat.nit
+++ b/lib/core/text/flat.nit
@@ -278,6 +278,23 @@ redef class FlatText
        end
 
        redef fun [](index) do return _items.char_at(char_to_byte_index(index))
+
+       # If `self` contains only digits and alpha <= 'f', return the corresponding integer.
+       #
+       #     assert "ff".to_hex == 255
+       redef fun to_hex(pos, ln) do
+               var res = 0
+               if pos == null then pos = 0
+               if ln == null then ln = length - pos
+               pos = char_to_byte_index(pos)
+               var its = _items
+               var max = pos + ln
+               for i in [pos .. max[ do
+                       res <<= 4
+                       res += its[i].ascii.from_hex
+               end
+               return res
+       end
 end
 
 # Immutable strings of characters.

diff --git a/lib/json/static.nit b/lib/json/static.nit
index 15a4355..7c0e2cf 100644 (file)
--- a/lib/json/static.nit
+++ b/lib/json/static.nit
@@ -138,7 +138,8 @@ redef class Text
        protected fun json_to_nit_string: String do
                var res = new FlatBuffer.with_capacity(bytelen)
                var i = 0
-               while i < self.length do
+               var ln = self.length
+               while i < ln do
                        var char = self[i]
                        if char == '\\' then
                                i += 1
@@ -154,21 +155,19 @@ redef class Text
                                else if char == 't' then
                                        char = '\t'
                                else if char == 'u' then
-                                       var code = substring(i + 1, 4)
-                                       var hx = code.to_hex
-                                       if hx >= 0xD800 and hx <= 0xDFFF then
-                                               var lostr = substring(i + 7, 4)
-                                               if lostr.length < 4 then
-                                                       hx = 0xFFFD
+                                       var u16_esc = from_utf16_digit(i + 1)
+                                       char = u16_esc.code_point
+                                       if char.is_surrogate and i + 10 < ln then
+                                               if self[i + 5] == '\\' and self[i + 6] == 'u' then
+                                                       u16_esc <<= 16
+                                                       u16_esc += from_utf16_digit(i + 7)
+                                                       char = u16_esc.from_utf16_surr.code_point
+                                                       i += 6
                                                else
-                                                       hx <<= 16
-                                                       hx += lostr.to_hex
-                                                       hx = hx.from_utf16_surr
+                                                       char = 0xFFFD.code_point
                                                end
-                                               i += 6
                                        end
                                        i += 4
-                                       char = hx.code_point
                                end
                                # `"`, `/` or `\` => Keep `char` as-is.
                        end