- cxon: utf-8 validation with DFA (Höhrmann/Felker)

Author: oknenavin

src/cxon/lang/common/cio/char.hxx

@@ -305,7 +305,7 @@ namespace cxon { namespace cio { namespace chr {
         }
 
     template <typename II>
-        inline int utf8_check(II i, II e) {
+        inline int utf8_check1(II i, II e) {
             // http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf
             // p41, Table 3-7. Well-Formed UTF-8 Byte Sequences
             CXON_ASSERT(i != e, "unexpected");
@@ -372,6 +372,44 @@ namespace cxon { namespace cio { namespace chr {
             return 0;
         }
 
+    namespace imp {
+        // https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+        // Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+        // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+        // TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+        // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+        // CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+        // IN THE SOFTWARE.
+        static const unsigned char utf8_decode_dfa_[] = {
+            11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,
+            11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,
+            11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,
+            11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11, 11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,
+             1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+             7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,  7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+             8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+            10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+            /////////////////////////////////////////////////////////////////////////////////////////////////
+             0,12,24,36,60,96,84,12,12,12,48,72,12,12,12,12, 12,12,12,12,12,12,12,12,12, 0,12,12,12,12,12, 0,
+            12, 0,12,12,12,24,12,12,12,12,12,24,12,24,12,12, 12,12,12,12,12,12,12,24,12,12,12,12,12,24,12,12,
+            12,12,12,12,12,24,12,12,12,12,12,12,12,12,12,36, 12,36,12,12,12,36,12,12,12,12,12,36,12,36,12,12,
+            12,36,12,12,12,12,12,12,12,12,12,12,12
+        };
+        inline unsigned utf8_decode_(unsigned state, unsigned byte) {
+            return utf8_decode_dfa_[256 + state + utf8_decode_dfa_[byte]];
+        }
+    }
+    template <typename II>
+        inline int utf8_check(II b, II e) {
+            unsigned s = 0;
+                for (II i = b; i != e; ++i) {
+                    s = imp::utf8_decode_(s, (unsigned char)*i);
+                    if (s ==  0) return int(i + 1 - b);
+                    if (s == 12) return 0;
+                }
+            return 0;
+        }
+
 }}}
 
 namespace cxon { namespace cio { namespace chr {