Damien George | 04b9147 | 2014-05-03 23:27:38 +0100 | [diff] [blame] | 1 | /* |
Alexander Steffen | 55f3324 | 2017-06-30 09:22:17 +0200 | [diff] [blame^] | 2 | * This file is part of the MicroPython project, http://micropython.org/ |
Damien George | 04b9147 | 2014-05-03 23:27:38 +0100 | [diff] [blame] | 3 | * |
| 4 | * The MIT License (MIT) |
| 5 | * |
| 6 | * Copyright (c) 2013, 2014 Damien P. George |
| 7 | * |
| 8 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 9 | * of this software and associated documentation files (the "Software"), to deal |
| 10 | * in the Software without restriction, including without limitation the rights |
| 11 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 12 | * copies of the Software, and to permit persons to whom the Software is |
| 13 | * furnished to do so, subject to the following conditions: |
| 14 | * |
| 15 | * The above copyright notice and this permission notice shall be included in |
| 16 | * all copies or substantial portions of the Software. |
| 17 | * |
| 18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 21 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 22 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 23 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 24 | * THE SOFTWARE. |
| 25 | */ |
| 26 | |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 27 | #include <stdint.h> |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 28 | |
Damien George | 51dfcb4 | 2015-01-01 20:27:54 +0000 | [diff] [blame] | 29 | #include "py/unicode.h" |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 30 | |
| 31 | // attribute flags |
| 32 | #define FL_PRINT (0x01) |
| 33 | #define FL_SPACE (0x02) |
| 34 | #define FL_DIGIT (0x04) |
| 35 | #define FL_ALPHA (0x08) |
| 36 | #define FL_UPPER (0x10) |
| 37 | #define FL_LOWER (0x20) |
Dave Hylands | afaa66b | 2015-05-19 16:41:07 -0700 | [diff] [blame] | 38 | #define FL_XDIGIT (0x40) |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 39 | |
| 40 | // shorthand character attributes |
| 41 | #define AT_PR (FL_PRINT) |
| 42 | #define AT_SP (FL_SPACE | FL_PRINT) |
Dave Hylands | afaa66b | 2015-05-19 16:41:07 -0700 | [diff] [blame] | 43 | #define AT_DI (FL_DIGIT | FL_PRINT | FL_XDIGIT) |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 44 | #define AT_AL (FL_ALPHA | FL_PRINT) |
| 45 | #define AT_UP (FL_UPPER | FL_ALPHA | FL_PRINT) |
| 46 | #define AT_LO (FL_LOWER | FL_ALPHA | FL_PRINT) |
Dave Hylands | afaa66b | 2015-05-19 16:41:07 -0700 | [diff] [blame] | 47 | #define AT_UX (FL_UPPER | FL_ALPHA | FL_PRINT | FL_XDIGIT) |
| 48 | #define AT_LX (FL_LOWER | FL_ALPHA | FL_PRINT | FL_XDIGIT) |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 49 | |
| 50 | // table of attributes for ascii characters |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 51 | STATIC const uint8_t attr[] = { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 52 | 0, 0, 0, 0, 0, 0, 0, 0, |
Damien George | 175cecf | 2014-04-10 11:39:36 +0100 | [diff] [blame] | 53 | 0, AT_SP, AT_SP, AT_SP, AT_SP, AT_SP, 0, 0, |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 54 | 0, 0, 0, 0, 0, 0, 0, 0, |
| 55 | 0, 0, 0, 0, 0, 0, 0, 0, |
| 56 | AT_SP, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, |
| 57 | AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, |
| 58 | AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, AT_DI, |
| 59 | AT_DI, AT_DI, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, |
Dave Hylands | afaa66b | 2015-05-19 16:41:07 -0700 | [diff] [blame] | 60 | AT_PR, AT_UX, AT_UX, AT_UX, AT_UX, AT_UX, AT_UX, AT_UP, |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 61 | AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, |
| 62 | AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, AT_UP, |
| 63 | AT_UP, AT_UP, AT_UP, AT_PR, AT_PR, AT_PR, AT_PR, AT_PR, |
Dave Hylands | afaa66b | 2015-05-19 16:41:07 -0700 | [diff] [blame] | 64 | AT_PR, AT_LX, AT_LX, AT_LX, AT_LX, AT_LX, AT_LX, AT_LO, |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 65 | AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, |
| 66 | AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, AT_LO, |
| 67 | AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0 |
| 68 | }; |
| 69 | |
Paul Sokolovsky | 1044c3d | 2014-06-14 06:39:20 +0300 | [diff] [blame] | 70 | // TODO: Rename to str_get_char |
| 71 | unichar utf8_get_char(const byte *s) { |
| 72 | #if MICROPY_PY_BUILTINS_STR_UNICODE |
Chris Angelico | c88987c | 2014-06-04 05:28:12 +1000 | [diff] [blame] | 73 | unichar ord = *s++; |
| 74 | if (!UTF8_IS_NONASCII(ord)) return ord; |
| 75 | ord &= 0x7F; |
| 76 | for (unichar mask = 0x40; ord & mask; mask >>= 1) { |
| 77 | ord &= ~mask; |
| 78 | } |
| 79 | while (UTF8_IS_CONT(*s)) { |
| 80 | ord = (ord << 6) | (*s++ & 0x3F); |
| 81 | } |
| 82 | return ord; |
Paul Sokolovsky | 1044c3d | 2014-06-14 06:39:20 +0300 | [diff] [blame] | 83 | #else |
| 84 | return *s; |
| 85 | #endif |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 86 | } |
| 87 | |
Paul Sokolovsky | 1044c3d | 2014-06-14 06:39:20 +0300 | [diff] [blame] | 88 | // TODO: Rename to str_next_char |
| 89 | const byte *utf8_next_char(const byte *s) { |
| 90 | #if MICROPY_PY_BUILTINS_STR_UNICODE |
Chris Angelico | c88987c | 2014-06-04 05:28:12 +1000 | [diff] [blame] | 91 | ++s; |
| 92 | while (UTF8_IS_CONT(*s)) { |
| 93 | ++s; |
| 94 | } |
Paul Sokolovsky | 1044c3d | 2014-06-14 06:39:20 +0300 | [diff] [blame] | 95 | return s; |
| 96 | #else |
| 97 | return s + 1; |
| 98 | #endif |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 99 | } |
| 100 | |
Damien George | 5318cc0 | 2014-12-10 22:37:07 +0000 | [diff] [blame] | 101 | mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) { |
Damien George | 40f3c02 | 2014-07-03 13:25:24 +0100 | [diff] [blame] | 102 | mp_uint_t i = 0; |
Paul Sokolovsky | 46d31e9 | 2014-06-14 03:16:17 +0300 | [diff] [blame] | 103 | while (ptr > s) { |
| 104 | if (!UTF8_IS_CONT(*--ptr)) { |
| 105 | i++; |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | return i; |
| 110 | } |
| 111 | |
Paul Sokolovsky | 9e215fa | 2014-06-28 23:14:30 +0300 | [diff] [blame] | 112 | // TODO: Rename to str_charlen |
Damien George | 4dea922 | 2015-04-09 15:29:54 +0000 | [diff] [blame] | 113 | mp_uint_t unichar_charlen(const char *str, mp_uint_t len) { |
Paul Sokolovsky | 1044c3d | 2014-06-14 06:39:20 +0300 | [diff] [blame] | 114 | #if MICROPY_PY_BUILTINS_STR_UNICODE |
Damien George | 40f3c02 | 2014-07-03 13:25:24 +0100 | [diff] [blame] | 115 | mp_uint_t charlen = 0; |
Chris Angelico | c88987c | 2014-06-04 05:28:12 +1000 | [diff] [blame] | 116 | for (const char *top = str + len; str < top; ++str) { |
| 117 | if (!UTF8_IS_CONT(*str)) { |
| 118 | ++charlen; |
| 119 | } |
| 120 | } |
| 121 | return charlen; |
Paul Sokolovsky | 1044c3d | 2014-06-14 06:39:20 +0300 | [diff] [blame] | 122 | #else |
| 123 | return len; |
| 124 | #endif |
Chris Angelico | c88987c | 2014-06-04 05:28:12 +1000 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | // Be aware: These unichar_is* functions are actually ASCII-only! |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 128 | bool unichar_isspace(unichar c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 129 | return c < 128 && (attr[c] & FL_SPACE) != 0; |
| 130 | } |
| 131 | |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 132 | bool unichar_isalpha(unichar c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 133 | return c < 128 && (attr[c] & FL_ALPHA) != 0; |
| 134 | } |
| 135 | |
Damien George | afc5063 | 2016-12-28 17:50:10 +1100 | [diff] [blame] | 136 | /* unused |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 137 | bool unichar_isprint(unichar c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 138 | return c < 128 && (attr[c] & FL_PRINT) != 0; |
| 139 | } |
Damien George | afc5063 | 2016-12-28 17:50:10 +1100 | [diff] [blame] | 140 | */ |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 141 | |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 142 | bool unichar_isdigit(unichar c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 143 | return c < 128 && (attr[c] & FL_DIGIT) != 0; |
| 144 | } |
| 145 | |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 146 | bool unichar_isxdigit(unichar c) { |
Dave Hylands | afaa66b | 2015-05-19 16:41:07 -0700 | [diff] [blame] | 147 | return c < 128 && (attr[c] & FL_XDIGIT) != 0; |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 148 | } |
| 149 | |
Alex March | 69d9e7d | 2016-02-16 13:36:18 +0000 | [diff] [blame] | 150 | bool unichar_isident(unichar c) { |
| 151 | return c < 128 && ((attr[c] & (FL_ALPHA | FL_DIGIT)) != 0 || c == '_'); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 152 | } |
| 153 | |
Damien George | c59af52 | 2014-05-11 17:53:11 +0100 | [diff] [blame] | 154 | bool unichar_isupper(unichar c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 155 | return c < 128 && (attr[c] & FL_UPPER) != 0; |
| 156 | } |
| 157 | |
Damien George | c59af52 | 2014-05-11 17:53:11 +0100 | [diff] [blame] | 158 | bool unichar_islower(unichar c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 159 | return c < 128 && (attr[c] & FL_LOWER) != 0; |
| 160 | } |
Paul Sokolovsky | 6913521 | 2014-05-10 19:47:41 +0300 | [diff] [blame] | 161 | |
| 162 | unichar unichar_tolower(unichar c) { |
Damien George | c59af52 | 2014-05-11 17:53:11 +0100 | [diff] [blame] | 163 | if (unichar_isupper(c)) { |
Paul Sokolovsky | 6913521 | 2014-05-10 19:47:41 +0300 | [diff] [blame] | 164 | return c + 0x20; |
| 165 | } |
| 166 | return c; |
| 167 | } |
| 168 | |
| 169 | unichar unichar_toupper(unichar c) { |
Damien George | c59af52 | 2014-05-11 17:53:11 +0100 | [diff] [blame] | 170 | if (unichar_islower(c)) { |
Paul Sokolovsky | 6913521 | 2014-05-10 19:47:41 +0300 | [diff] [blame] | 171 | return c - 0x20; |
| 172 | } |
| 173 | return c; |
| 174 | } |
Dave Hylands | 3ad94d6 | 2015-05-18 14:41:25 -0700 | [diff] [blame] | 175 | |
| 176 | mp_uint_t unichar_xdigit_value(unichar c) { |
| 177 | // c is assumed to be hex digit |
| 178 | mp_uint_t n = c - '0'; |
| 179 | if (n > 9) { |
| 180 | n &= ~('a' - 'A'); |
| 181 | n -= ('A' - ('9' + 1)); |
| 182 | } |
| 183 | return n; |
| 184 | } |